From 038fd04a825312cf2319ae9e33b6cdfdcf4f5abb Mon Sep 17 00:00:00 2001 From: pyalex Date: Thu, 6 Jan 2022 16:51:11 +0100 Subject: [PATCH 01/17] Validating historical features against reference dataset Signed-off-by: pyalex --- docs/reference/dqm.md | 79 ++++++++++ protos/feast/core/ValidationProfile.proto | 39 +++++ protos/feast/core/ValidationReference.proto | 35 +++++ sdk/python/feast/dqm/__init__.py | 0 sdk/python/feast/dqm/errors.py | 13 ++ sdk/python/feast/dqm/profilers/__init__.py | 0 sdk/python/feast/dqm/profilers/ge_profiler.py | 126 ++++++++++++++++ sdk/python/feast/dqm/profilers/profiler.py | 88 +++++++++++ sdk/python/feast/dqm/utils.py | 73 ++++++++++ sdk/python/feast/feature_store.py | 10 +- .../feast/infra/offline_stores/bigquery.py | 4 +- .../feast/infra/offline_stores/redshift.py | 2 +- sdk/python/feast/saved_dataset.py | 20 +++ sdk/python/setup.py | 8 +- sdk/python/tests/foo_provider.py | 1 + .../tests/integration/e2e/test_validation.py | 137 ++++++++++++++++++ 16 files changed, 631 insertions(+), 4 deletions(-) create mode 100644 docs/reference/dqm.md create mode 100644 protos/feast/core/ValidationProfile.proto create mode 100644 protos/feast/core/ValidationReference.proto create mode 100644 sdk/python/feast/dqm/__init__.py create mode 100644 sdk/python/feast/dqm/errors.py create mode 100644 sdk/python/feast/dqm/profilers/__init__.py create mode 100644 sdk/python/feast/dqm/profilers/ge_profiler.py create mode 100644 sdk/python/feast/dqm/profilers/profiler.py create mode 100644 sdk/python/feast/dqm/utils.py create mode 100644 sdk/python/tests/integration/e2e/test_validation.py diff --git a/docs/reference/dqm.md b/docs/reference/dqm.md new file mode 100644 index 00000000000..3f49c06274a --- /dev/null +++ b/docs/reference/dqm.md @@ -0,0 +1,79 @@ +# Data Quality Monitoring + +Data Quality Monitoring (DQM) is a Feast module aimed to help users to validate their data with the user-curated set of rules. +Validation could be applied during: +* Historical retrieval (training dataset generation) +* [planned] Writing features into an online store +* [planned] Reading features from an online store + +Its goal is to address several complex data problems, namely: +* Data Consistency - new training dataset could be significantly different from previous, which will require change in model architecture. +* Issues/bugs in upstream pipeline - bug in upstream could case invalid values to overwrite existing valid values in an online store. +* Training/serving skew - distribution shift could significantly decrease performance of the model. + +> By “monitoring data quality” we understand verifying that the characteristics of tested dataset (we call it dataset's profile) are "equivalent" to the characteristics of reference dataset. +> Eg, data currently passed to the model hasn’t changed significantly since the model was trained and expectations implicitly made by ML algorithm during training are still met. +> How exactly profiles equivalency should be measured is up to the user. + +### Overview + +Validation process consists of the next steps: +1. User prepares reference dataset (currently only [saved dataset](../getting-started/concepts/dataset.md) from historical retrieval is supported). +2. User defines profiler function, which should produce profile by given dataset. +3. Validation of tested dataset is performed with reference dataset and profiler provided as parameters. + +### Preparations +Feast with DQM support can be installed via +```shell +pip install 'feast[dqm]' +``` + +### Dataset profile +Currently, Feast supports only [great expectation's](https://greatexpectations.io/) [ExpectationSuite](https://legacy.docs.greatexpectations.io/en/latest/autoapi/great_expectations/core/expectation_suite/index.html#great_expectations.core.expectation_suite.ExpectationSuite) +as dataset's profile. Hence, user needs to define a function (profiler) that would receive a dataset and return [ExpectationSuite](https://legacy.docs.greatexpectations.io/en/latest/autoapi/great_expectations/core/expectation_suite/index.html#great_expectations.core.expectation_suite.ExpectationSuite). + +Either automatic profiler or user selected expectations could be used in profiler function: +```python +from great_expectations.dataset import Dataset +from great_expectations.core.expectation_suite import ExpectationSuite + +from feast.dqm.profilers.ge_profiler import ge_profiler + +@ge_profiler +def automatic_profiler(dataset: Dataset) -> ExpectationSuite: + from great_expectations.profile.user_configurable_profiler import UserConfigurableProfiler + + return UserConfigurableProfiler( + profile_dataset=dataset, + ignored_columns=['conv_rate'], + value_set_threshold='few' + ).build_suite() +``` + +```python +@ge_profiler +def manual_profiler(dataset: Dataset) -> ExpectationSuite: + dataset.expect_column_max_to_be_between("column", 1, 2) + return dataset.get_expectation_suite() +``` + + + +### Validating Training Dataset +During retrieval of historical features additional parameter `validation_reference` could be passed. +If this parameter is supplied `get_historical_features` will return `RetrievalJobWithValidation` instead of simple `RetrievalJob`. +Such job will run validation once dataset is materialized (when `.to_df()` or `.to_arrow()` called). In case if validation successful materialized dataset is returned (no change to previous/regular behavior). +Otherwise `feast.dqm.errors.ValidationFailed` exception would be raised. It will consist of all details for expectations that didn't pass. + +```python +from feast import FeatureStore + +fs = FeatureStore(".") + +fs.get_historical_features( + ..., + validation_reference=fs + .get_saved_dataset("my_reference_dataset") + .as_reference(profiler=manual_profiler) +) +``` diff --git a/protos/feast/core/ValidationProfile.proto b/protos/feast/core/ValidationProfile.proto new file mode 100644 index 00000000000..4c67b263213 --- /dev/null +++ b/protos/feast/core/ValidationProfile.proto @@ -0,0 +1,39 @@ +// +// Copyright 2021 The Feast Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + + +syntax = "proto3"; + +package feast.core; +option java_package = "feast.proto.core"; +option java_outer_classname = "SavedDatasetProto"; +option go_package = "github.com/feast-dev/feast/sdk/go/protos/feast/core"; + +import "google/protobuf/timestamp.proto"; + +message GEValidationProfiler { + message UserDefinedProfiler { + // The python-syntax function body (serialized by dill) + bytes body = 1; + } + + UserDefinedProfiler profiler = 1; +} + +message GEValidationProfile { + // JSON-serialized ExpectationSuite object + bytes expectation_suite = 1; +} diff --git a/protos/feast/core/ValidationReference.proto b/protos/feast/core/ValidationReference.proto new file mode 100644 index 00000000000..56faddfbf1d --- /dev/null +++ b/protos/feast/core/ValidationReference.proto @@ -0,0 +1,35 @@ +// +// Copyright 2021 The Feast Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + + +syntax = "proto3"; + +package feast.core; +option java_package = "feast.proto.core"; +option java_outer_classname = "SavedDatasetProto"; +option go_package = "github.com/feast-dev/feast/sdk/go/protos/feast/core"; + +import "feast/core/SavedDataset.proto"; +import "feast/core/ValidationProfile.proto"; + + +message ValidationReference { + SavedDataset dataset = 1; + + oneof profiler { + GEValidationProfiler ge_profiler = 2; + } +} \ No newline at end of file diff --git a/sdk/python/feast/dqm/__init__.py b/sdk/python/feast/dqm/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/sdk/python/feast/dqm/errors.py b/sdk/python/feast/dqm/errors.py new file mode 100644 index 00000000000..c4179f72b3c --- /dev/null +++ b/sdk/python/feast/dqm/errors.py @@ -0,0 +1,13 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from .profilers.profiler import ValidationReport + + +class ValidationFailed(Exception): + def __init__(self, validation_report: "ValidationReport"): + self.validation_report = validation_report + + @property + def report(self) -> "ValidationReport": + return self.validation_report diff --git a/sdk/python/feast/dqm/profilers/__init__.py b/sdk/python/feast/dqm/profilers/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/sdk/python/feast/dqm/profilers/ge_profiler.py b/sdk/python/feast/dqm/profilers/ge_profiler.py new file mode 100644 index 00000000000..d90e50eaf3f --- /dev/null +++ b/sdk/python/feast/dqm/profilers/ge_profiler.py @@ -0,0 +1,126 @@ +import json +from typing import Any, Callable, Dict, List + +import dill +import great_expectations as ge +import numpy as np +import pandas as pd +from great_expectations.core import ExpectationSuite +from great_expectations.dataset import PandasDataset +from great_expectations.profile.base import ProfilerTypeMapping + +from feast.dqm.profilers.profiler import ( + Profile, + Profiler, + ValidationError, + ValidationReport, +) +from feast.protos.feast.core.ValidationProfile_pb2 import ( + GEValidationProfile as GEValidationProfileProto, +) +from feast.protos.feast.core.ValidationProfile_pb2 import ( + GEValidationProfiler as GEValidationProfilerProto, +) + + +def _prepare_dataset(dataset): + for column in dataset.columns: + if dataset.expect_column_values_to_be_in_type_list( + column, type_list=sorted(list(ProfilerTypeMapping.DATETIME_TYPE_NAMES)) + ).success: + + # GE cannot parse Timestamp or other pandas datetime time + dataset[column] = dataset[column].dt.strftime("%Y-%m-%dT%H:%M:%S") + + if dataset[column].dtype == np.float32: + # GE converts expectation arguments into native Python float + # This could cause error on comparison => so better to convert to double prematurely + dataset[column] = dataset[column].astype(np.float64) + + +class GEProfile(Profile): + expectation_suite: ExpectationSuite + + def __init__(self, expectation_suite: ExpectationSuite): + self.expectation_suite = expectation_suite + + def validate(self, df: pd.DataFrame) -> "GEValidationReport": + dataset = PandasDataset(df) + + _prepare_dataset(dataset) + + results = ge.validate( + dataset, expectation_suite=self.expectation_suite, result_format="COMPLETE" + ) + return GEValidationReport(results) + + def to_proto(self): + return GEValidationProfileProto( + expectation_suite=json.dumps(self.expectation_suite.to_json_dict()).encode() + ) + + @classmethod + def from_proto(cls, proto: GEValidationProfileProto) -> "GEProfile": + return GEProfile( + expectation_suite=ExpectationSuite(**json.loads(proto.expectation_suite)) + ) + + +class GEProfiler(Profiler): + def __init__( + self, user_defined_profiler: Callable[[pd.DataFrame], ExpectationSuite] + ): + self.user_defined_profiler = user_defined_profiler + + def analyze_dataset(self, df: pd.DataFrame) -> Profile: + dataset = PandasDataset(df) + + _prepare_dataset(dataset) + + return GEProfile(expectation_suite=self.user_defined_profiler(dataset)) + + def to_proto(self): + return GEValidationProfilerProto( + profiler=GEValidationProfilerProto.UserDefinedProfiler( + body=dill.dumps(self.user_defined_profiler, recurse=True) + ) + ) + + @classmethod + def from_proto(cls, proto: GEValidationProfilerProto) -> "GEProfiler": + return GEProfiler(user_defined_profiler=dill.loads(proto.profiler.body)) + + +class GEValidationReport(ValidationReport): + def __init__(self, validation_result: Dict[Any, Any]): + self._validation_result = validation_result + + @property + def is_success(self) -> bool: + return self._validation_result["success"] + + @property + def errors(self) -> List["ValidationError"]: + return [ + ValidationError( + check_name=res.expectation_config.expectation_type, + column_name=res.expectation_config.kwargs["column"], + check_config=res.expectation_config.kwargs, + missing_count=res["result"].get("missing_count"), + missing_percent=res["result"].get("missing_percent"), + ) + for res in self._validation_result["results"] + if not res["success"] + ] + + def __repr__(self): + failed_expectations = [ + res.to_json_dict() + for res in self._validation_result["results"] + if not res["success"] + ] + return json.dumps(failed_expectations, indent=2) + + +def ge_profiler(func): + return GEProfiler(user_defined_profiler=func) diff --git a/sdk/python/feast/dqm/profilers/profiler.py b/sdk/python/feast/dqm/profilers/profiler.py new file mode 100644 index 00000000000..5d2e9d36bc1 --- /dev/null +++ b/sdk/python/feast/dqm/profilers/profiler.py @@ -0,0 +1,88 @@ +import abc +from typing import Any, List, Optional + +import pandas as pd + + +class Profile: + @abc.abstractmethod + def validate(self, dataset: pd.DataFrame) -> "ValidationReport": + """ + Run set of rules / expectations from current profile against given dataset. + + Return ValidationReport + """ + ... + + @abc.abstractmethod + def to_proto(self): + ... + + @classmethod + @abc.abstractmethod + def from_proto(cls, proto) -> "Profile": + ... + + +class Profiler: + @abc.abstractmethod + def analyze_dataset(self, dataset: pd.DataFrame) -> Profile: + """ + Generate Profile object with dataset's characteristics (with rules / expectations) + from given dataset (as pandas dataframe). + """ + ... + + @abc.abstractmethod + def to_proto(self): + ... + + @classmethod + @abc.abstractmethod + def from_proto(cls, proto) -> "Profiler": + ... + + +class ValidationReport: + @property + @abc.abstractmethod + def is_success(self) -> bool: + """ + Return whether validation was successful + """ + ... + + @property + @abc.abstractmethod + def errors(self) -> List["ValidationError"]: + """ + Return list of ValidationErrors if validation failed (is_success = false) + """ + ... + + +class ValidationError: + check_name: str + column_name: str + + check_config: Optional[Any] + + missing_count: Optional[int] + missing_percent: Optional[float] + + def __init__( + self, + check_name: str, + column_name: str, + check_config: Optional[Any] = None, + missing_count: Optional[int] = None, + missing_percent: Optional[float] = None, + ): + self.check_name = check_name + self.column_name = column_name + self.check_config = check_config + self.missing_count = missing_count + self.missing_percent = missing_percent + + def __repr__(self): + return f"" diff --git a/sdk/python/feast/dqm/utils.py b/sdk/python/feast/dqm/utils.py new file mode 100644 index 00000000000..7e306f8019d --- /dev/null +++ b/sdk/python/feast/dqm/utils.py @@ -0,0 +1,73 @@ +from typing import TYPE_CHECKING, List, Optional + +import pandas as pd +import pyarrow + +from feast.dqm.errors import ValidationFailed +from feast.dqm.profilers.profiler import Profile +from feast.infra.offline_stores.offline_store import RetrievalJob, RetrievalMetadata +from feast.saved_dataset import SavedDatasetStorage, ValidationReference + +if TYPE_CHECKING: + from feast.feature_store import FeatureStore + from feast.on_demand_feature_view import OnDemandFeatureView + + +class RetrievalJobWithValidation(RetrievalJob): + def __init__( + self, + retrieval_job: RetrievalJob, + validation_reference: ValidationReference, + feature_store: "FeatureStore", + ): + self._retrieval_job = retrieval_job + self._validation_reference = validation_reference + self._feature_store = feature_store + + def to_df(self) -> pd.DataFrame: + df = self._retrieval_job.to_df() + + profile = get_reference_profile(self._feature_store, self._validation_reference) + validation_result = profile.validate(df) + if not validation_result.is_success: + raise ValidationFailed(validation_result) + + return df + + def to_arrow(self) -> pyarrow.Table: + table = self._retrieval_job.to_arrow() + + profile = get_reference_profile(self._feature_store, self._validation_reference) + validation_result = profile.validate(table.to_pandas()) + if not validation_result.is_success: + raise ValidationFailed(validation_result) + + return table + + @property + def full_feature_names(self) -> bool: + return self._retrieval_job.full_feature_names + + @property + def on_demand_feature_views(self) -> Optional[List["OnDemandFeatureView"]]: + return self._retrieval_job.on_demand_feature_views + + def _to_df_internal(self) -> pd.DataFrame: + raise NotImplemented + + def _to_arrow_internal(self) -> pyarrow.Table: + raise NotImplemented + + def persist(self, storage: SavedDatasetStorage) -> "RetrievalJob": + return self._retrieval_job.persist(storage) + + @property + def metadata(self) -> Optional[RetrievalMetadata]: + return self._retrieval_job.metadata + + +def get_reference_profile( + fs: "FeatureStore", reference: ValidationReference +) -> Profile: + dataset = fs.get_saved_dataset(reference.dataset.name) + return reference.profiler.analyze_dataset(dataset.to_df()) diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index 6b1dadde5c9..fcd2e4f90f9 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -41,6 +41,7 @@ from feast.base_feature_view import BaseFeatureView from feast.diff.infra_diff import InfraDiff, diff_infra_protos from feast.diff.registry_diff import RegistryDiff, apply_diff_to_registry, diff_between +from feast.dqm.utils import RetrievalJobWithValidation from feast.entity import Entity from feast.errors import ( EntityNotFoundException, @@ -78,7 +79,7 @@ from feast.repo_config import RepoConfig, load_repo_config from feast.repo_contents import RepoContents from feast.request_feature_view import RequestFeatureView -from feast.saved_dataset import SavedDataset, SavedDatasetStorage +from feast.saved_dataset import SavedDataset, SavedDatasetStorage, ValidationReference from feast.type_map import python_values_to_proto_values from feast.usage import log_exceptions, log_exceptions_and_usage, set_usage_attribute from feast.value_type import ValueType @@ -709,6 +710,7 @@ def get_historical_features( entity_df: Union[pd.DataFrame, str], features: Union[List[str], FeatureService], full_feature_names: bool = False, + validation_reference: Optional[ValidationReference] = None, ) -> RetrievalJob: """Enrich an entity dataframe with historical feature values for either training or batch scoring. @@ -734,6 +736,7 @@ def get_historical_features( full_feature_names: A boolean that provides the option to add the feature view prefixes to the feature names, changing them from the format "feature" to "feature_view__feature" (e.g., "daily_transactions" changes to "customer_fv__daily_transactions"). By default, this value is set to False. + validation_reference: If provided resulting dataset will be validated against this reference profile. Returns: RetrievalJob which can be used to materialize the results. @@ -823,6 +826,11 @@ def get_historical_features( full_feature_names, ) + if validation_reference: + job = RetrievalJobWithValidation( + job, validation_reference, feature_store=self + ) + return job @log_exceptions_and_usage diff --git a/sdk/python/feast/infra/offline_stores/bigquery.py b/sdk/python/feast/infra/offline_stores/bigquery.py index 44e62d6ad1a..21512120a3c 100644 --- a/sdk/python/feast/infra/offline_stores/bigquery.py +++ b/sdk/python/feast/infra/offline_stores/bigquery.py @@ -100,7 +100,9 @@ def pull_latest_from_table_or_query( if created_timestamp_column: timestamps.append(created_timestamp_column) timestamp_desc_string = " DESC, ".join(timestamps) + " DESC" - field_string = ", ".join(join_key_columns + feature_name_columns + timestamps) + field_string = ", ".join( + set(join_key_columns + feature_name_columns + timestamps) + ) client = _get_bigquery_client( project=config.offline_store.project_id, diff --git a/sdk/python/feast/infra/offline_stores/redshift.py b/sdk/python/feast/infra/offline_stores/redshift.py index 3efd45bc741..8d729719dac 100644 --- a/sdk/python/feast/infra/offline_stores/redshift.py +++ b/sdk/python/feast/infra/offline_stores/redshift.py @@ -91,7 +91,7 @@ def pull_latest_from_table_or_query( timestamp_columns.append(created_timestamp_column) timestamp_desc_string = " DESC, ".join(timestamp_columns) + " DESC" field_string = ", ".join( - join_key_columns + feature_name_columns + timestamp_columns + set(join_key_columns + feature_name_columns + timestamp_columns) ) redshift_client = aws_utils.get_redshift_data_client( diff --git a/sdk/python/feast/saved_dataset.py b/sdk/python/feast/saved_dataset.py index 39708685795..64dcffaf803 100644 --- a/sdk/python/feast/saved_dataset.py +++ b/sdk/python/feast/saved_dataset.py @@ -7,6 +7,7 @@ from google.protobuf.json_format import MessageToJson from feast.data_source import DataSource +from feast.dqm.profilers.profiler import Profile, Profiler from feast.protos.feast.core.SavedDataset_pb2 import SavedDataset as SavedDatasetProto from feast.protos.feast.core.SavedDataset_pb2 import SavedDatasetMeta, SavedDatasetSpec from feast.protos.feast.core.SavedDataset_pb2 import ( @@ -60,6 +61,8 @@ class SavedDataset: min_event_timestamp: Optional[datetime] = None max_event_timestamp: Optional[datetime] = None + _retrieval_job: Optional["RetrievalJob"] = None + def __init__( self, name: str, @@ -76,6 +79,8 @@ def __init__( self.full_feature_names = full_feature_names self.tags = tags or {} + self._retrieval_job = None + def __repr__(self): items = (f"{k} = {v}" for k, v in self.__dict__.items()) return f"<{self.__class__.__name__}({', '.join(items)})>" @@ -183,3 +188,18 @@ def to_arrow(self) -> pyarrow.Table: ) return self._retrieval_job.to_arrow() + + def as_reference(self, profiler: "Profiler") -> "ValidationReference": + return ValidationReference(profiler=profiler, dataset=self) + + def get_profile(self, profiler: Profiler) -> Profile: + return profiler.analyze_dataset(self.to_df()) + + +class ValidationReference: + dataset: SavedDataset + profiler: Profiler + + def __init__(self, dataset: SavedDataset, profiler: Profiler): + self.dataset = dataset + self.profiler = profiler diff --git a/sdk/python/setup.py b/sdk/python/setup.py index cb5381813b5..eda775d66b3 100644 --- a/sdk/python/setup.py +++ b/sdk/python/setup.py @@ -90,6 +90,10 @@ "snowflake-connector-python[pandas]>=2.7.3", ] +DQM_REQUIRED = [ + "great_expectations>=0.14.0" +] + CI_REQUIRED = ( [ "cryptography==3.3.2", @@ -135,6 +139,7 @@ + REDIS_REQUIRED + AWS_REQUIRED + SNOWFLAKE_REQUIRED + + DQM_REQUIRED ) DEV_REQUIRED = ["mypy-protobuf>=3.1.0", "grpcio-testing==1.*"] + CI_REQUIRED @@ -236,7 +241,8 @@ def run(self): "gcp": GCP_REQUIRED, "aws": AWS_REQUIRED, "redis": REDIS_REQUIRED, - "snowflake": SNOWFLAKE_REQUIRED + "snowflake": SNOWFLAKE_REQUIRED, + "dqm": DQM_REQUIRED, }, include_package_data=True, license="Apache", diff --git a/sdk/python/tests/foo_provider.py b/sdk/python/tests/foo_provider.py index 1d4ce7d6cb6..2abec37b930 100644 --- a/sdk/python/tests/foo_provider.py +++ b/sdk/python/tests/foo_provider.py @@ -65,6 +65,7 @@ def get_historical_features( registry: Registry, project: str, full_feature_names: bool = False, + save_as: Optional[SavedDataset] = None, ) -> RetrievalJob: pass diff --git a/sdk/python/tests/integration/e2e/test_validation.py b/sdk/python/tests/integration/e2e/test_validation.py new file mode 100644 index 00000000000..4edac317dee --- /dev/null +++ b/sdk/python/tests/integration/e2e/test_validation.py @@ -0,0 +1,137 @@ +import pandas as pd +import pytest +from great_expectations.core import ExpectationSuite +from great_expectations.dataset import PandasDataset + +from feast.dqm.errors import ValidationFailed +from feast.dqm.profilers.ge_profiler import ge_profiler +from feast.saved_dataset import SavedDatasetOptions +from tests.integration.feature_repos.repo_configuration import ( + construct_universal_feature_views, +) +from tests.integration.feature_repos.universal.entities import ( + customer, + driver, + location, +) + +_features = [ + "customer_profile:current_balance", + "customer_profile:avg_passenger_count", + "customer_profile:lifetime_trip_count", + "order:order_is_success", + "global_stats:num_rides", + "global_stats:avg_ride_length", +] + + +@ge_profiler +def configurable_profiler(dataset: PandasDataset) -> ExpectationSuite: + from great_expectations.profile.user_configurable_profiler import ( + UserConfigurableProfiler, + ) + + return UserConfigurableProfiler( + profile_dataset=dataset, + excluded_expectations=[ + "expect_table_columns_to_match_ordered_list", + "expect_table_row_count_to_be_between", + ], + value_set_threshold="few", + ).build_suite() + + +@ge_profiler +def profiler_with_unrealistic_expectations(dataset: PandasDataset) -> ExpectationSuite: + # need to create dataframe with corrupted data first + df = pd.DataFrame() + df["current_balance"] = [-100] + df["avg_passenger_count"] = [0] + + other_ds = PandasDataset(df) + other_ds.expect_column_max_to_be_between("current_balance", -1000, -100) + other_ds.expect_column_values_to_be_in_set("avg_passenger_count", value_set={0}) + + # this should pass + other_ds.expect_column_min_to_be_between("avg_passenger_count", 0, 1000) + + return other_ds.get_expectation_suite() + + +@pytest.mark.integration +@pytest.mark.universal +def test_historical_retrieval_with_validation(environment, universal_data_sources): + store = environment.feature_store + + (entities, datasets, data_sources) = universal_data_sources + feature_views = construct_universal_feature_views(data_sources) + + store.apply([driver(), customer(), location(), *feature_views.values()]) + + entity_df = datasets["entity"].drop( + columns=["order_id", "origin_id", "destination_id"] + ) + + store.get_historical_features( + entity_df=entity_df, + features=_features, + save_as=SavedDatasetOptions( + name="my_training_dataset", + storage=environment.data_source_creator.create_saved_dataset_destination(), + ), + ) + + job = store.get_historical_features( + entity_df=entity_df, + features=_features, + validation_reference=store.get_saved_dataset( + "my_training_dataset" + ).as_reference(profiler=configurable_profiler), + ) + + # if validation pass there will be no exceptions on this point + job.to_df() + + +@pytest.mark.integration +@pytest.mark.universal +def test_historical_retrieval_fails_on_validation(environment, universal_data_sources): + store = environment.feature_store + + (entities, datasets, data_sources) = universal_data_sources + feature_views = construct_universal_feature_views(data_sources) + + store.apply([driver(), customer(), location(), *feature_views.values()]) + + entity_df = datasets["entity"].drop( + columns=["order_id", "origin_id", "destination_id"] + ) + + store.get_historical_features( + entity_df=entity_df, + features=_features, + save_as=SavedDatasetOptions( + name="my_other_dataset", + storage=environment.data_source_creator.create_saved_dataset_destination(), + ), + ) + + job = store.get_historical_features( + entity_df=entity_df, + features=_features, + validation_reference=store.get_saved_dataset("my_other_dataset").as_reference( + profiler=profiler_with_unrealistic_expectations + ), + ) + + with pytest.raises(ValidationFailed) as exc_info: + job.to_df() + + failed_expectations = exc_info.value.report.errors + assert len(failed_expectations) == 2 + + assert failed_expectations[0].check_name == "expect_column_max_to_be_between" + assert failed_expectations[0].column_name == "current_balance" + + assert failed_expectations[1].check_name == "expect_column_values_to_be_in_set" + assert failed_expectations[1].column_name == "avg_passenger_count" From 3abda730bd1a127cb8744fe88fcb94da5c8e41a7 Mon Sep 17 00:00:00 2001 From: pyalex Date: Thu, 27 Jan 2022 15:05:44 +0700 Subject: [PATCH 02/17] fixes after rebase Signed-off-by: pyalex --- .../tests/integration/e2e/test_validation.py | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/sdk/python/tests/integration/e2e/test_validation.py b/sdk/python/tests/integration/e2e/test_validation.py index 4edac317dee..23a6001e7f6 100644 --- a/sdk/python/tests/integration/e2e/test_validation.py +++ b/sdk/python/tests/integration/e2e/test_validation.py @@ -5,7 +5,6 @@ from feast.dqm.errors import ValidationFailed from feast.dqm.profilers.ge_profiler import ge_profiler -from feast.saved_dataset import SavedDatasetOptions from tests.integration.feature_repos.repo_configuration import ( construct_universal_feature_views, ) @@ -72,13 +71,15 @@ def test_historical_retrieval_with_validation(environment, universal_data_source columns=["order_id", "origin_id", "destination_id"] ) - store.get_historical_features( + reference_job = store.get_historical_features( entity_df=entity_df, features=_features, - save_as=SavedDatasetOptions( - name="my_training_dataset", - storage=environment.data_source_creator.create_saved_dataset_destination(), - ), + ) + + store.create_saved_dataset( + from_=reference_job, + name="my_training_dataset", + storage=environment.data_source_creator.create_saved_dataset_destination(), ) job = store.get_historical_features( @@ -107,13 +108,15 @@ def test_historical_retrieval_fails_on_validation(environment, universal_data_so columns=["order_id", "origin_id", "destination_id"] ) - store.get_historical_features( + reference_job = store.get_historical_features( entity_df=entity_df, features=_features, - save_as=SavedDatasetOptions( - name="my_other_dataset", - storage=environment.data_source_creator.create_saved_dataset_destination(), - ), + ) + + store.create_saved_dataset( + from_=reference_job, + name="my_other_dataset", + storage=environment.data_source_creator.create_saved_dataset_destination(), ) job = store.get_historical_features( From d4cbe97556823e61f969588910f341cd7ab7d62f Mon Sep 17 00:00:00 2001 From: pyalex Date: Thu, 27 Jan 2022 15:21:48 +0700 Subject: [PATCH 03/17] fixes after rebase Signed-off-by: pyalex --- sdk/python/feast/dqm/utils.py | 4 ++-- sdk/python/tests/integration/e2e/test_validation.py | 6 ++---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/sdk/python/feast/dqm/utils.py b/sdk/python/feast/dqm/utils.py index 7e306f8019d..6238b38e4b5 100644 --- a/sdk/python/feast/dqm/utils.py +++ b/sdk/python/feast/dqm/utils.py @@ -53,10 +53,10 @@ def on_demand_feature_views(self) -> Optional[List["OnDemandFeatureView"]]: return self._retrieval_job.on_demand_feature_views def _to_df_internal(self) -> pd.DataFrame: - raise NotImplemented + raise NotImplementedError def _to_arrow_internal(self) -> pyarrow.Table: - raise NotImplemented + raise NotImplementedError def persist(self, storage: SavedDatasetStorage) -> "RetrievalJob": return self._retrieval_job.persist(storage) diff --git a/sdk/python/tests/integration/e2e/test_validation.py b/sdk/python/tests/integration/e2e/test_validation.py index 23a6001e7f6..3996e5fd0a4 100644 --- a/sdk/python/tests/integration/e2e/test_validation.py +++ b/sdk/python/tests/integration/e2e/test_validation.py @@ -72,8 +72,7 @@ def test_historical_retrieval_with_validation(environment, universal_data_source ) reference_job = store.get_historical_features( - entity_df=entity_df, - features=_features, + entity_df=entity_df, features=_features, ) store.create_saved_dataset( @@ -109,8 +108,7 @@ def test_historical_retrieval_fails_on_validation(environment, universal_data_so ) reference_job = store.get_historical_features( - entity_df=entity_df, - features=_features, + entity_df=entity_df, features=_features, ) store.create_saved_dataset( From 3d7fcea90c08e17a2c60fc418b7d07c305ed8876 Mon Sep 17 00:00:00 2001 From: pyalex Date: Thu, 27 Jan 2022 15:47:10 +0700 Subject: [PATCH 04/17] update ci requirements Signed-off-by: pyalex --- sdk/python/requirements/py3.8-requirements.txt | 2 +- sdk/python/requirements/py3.9-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/python/requirements/py3.8-requirements.txt b/sdk/python/requirements/py3.8-requirements.txt index e94fe117b4a..90b42760132 100644 --- a/sdk/python/requirements/py3.8-requirements.txt +++ b/sdk/python/requirements/py3.8-requirements.txt @@ -65,7 +65,7 @@ markupsafe==2.0.1 # via jinja2 mmh3==3.0.0 # via feast (setup.py) -numpy==1.22.1 +numpy==1.21.5 # via # pandas # pandavro diff --git a/sdk/python/requirements/py3.9-requirements.txt b/sdk/python/requirements/py3.9-requirements.txt index 187cb02154b..8db9fd4b14f 100644 --- a/sdk/python/requirements/py3.9-requirements.txt +++ b/sdk/python/requirements/py3.9-requirements.txt @@ -63,7 +63,7 @@ markupsafe==2.0.1 # via jinja2 mmh3==3.0.0 # via feast (setup.py) -numpy==1.22.1 +numpy==1.21.5 # via # pandas # pandavro From 07ae01287e1e11f1ade99b4a2ba37b9fa69647dd Mon Sep 17 00:00:00 2001 From: pyalex Date: Thu, 27 Jan 2022 16:00:11 +0700 Subject: [PATCH 05/17] some reverts Signed-off-by: pyalex --- sdk/python/feast/infra/offline_stores/bigquery.py | 2 +- sdk/python/feast/infra/offline_stores/redshift.py | 2 +- sdk/python/tests/foo_provider.py | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/sdk/python/feast/infra/offline_stores/bigquery.py b/sdk/python/feast/infra/offline_stores/bigquery.py index 21512120a3c..288b032f348 100644 --- a/sdk/python/feast/infra/offline_stores/bigquery.py +++ b/sdk/python/feast/infra/offline_stores/bigquery.py @@ -101,7 +101,7 @@ def pull_latest_from_table_or_query( timestamps.append(created_timestamp_column) timestamp_desc_string = " DESC, ".join(timestamps) + " DESC" field_string = ", ".join( - set(join_key_columns + feature_name_columns + timestamps) + join_key_columns + feature_name_columns + timestamps ) client = _get_bigquery_client( diff --git a/sdk/python/feast/infra/offline_stores/redshift.py b/sdk/python/feast/infra/offline_stores/redshift.py index 8d729719dac..3efd45bc741 100644 --- a/sdk/python/feast/infra/offline_stores/redshift.py +++ b/sdk/python/feast/infra/offline_stores/redshift.py @@ -91,7 +91,7 @@ def pull_latest_from_table_or_query( timestamp_columns.append(created_timestamp_column) timestamp_desc_string = " DESC, ".join(timestamp_columns) + " DESC" field_string = ", ".join( - set(join_key_columns + feature_name_columns + timestamp_columns) + join_key_columns + feature_name_columns + timestamp_columns ) redshift_client = aws_utils.get_redshift_data_client( diff --git a/sdk/python/tests/foo_provider.py b/sdk/python/tests/foo_provider.py index 2abec37b930..1d4ce7d6cb6 100644 --- a/sdk/python/tests/foo_provider.py +++ b/sdk/python/tests/foo_provider.py @@ -65,7 +65,6 @@ def get_historical_features( registry: Registry, project: str, full_feature_names: bool = False, - save_as: Optional[SavedDataset] = None, ) -> RetrievalJob: pass From 2121ae57a9bec7a4124e8962f4cc6d7496043ba5 Mon Sep 17 00:00:00 2001 From: pyalex Date: Thu, 27 Jan 2022 16:04:34 +0700 Subject: [PATCH 06/17] format Signed-off-by: pyalex --- sdk/python/feast/infra/offline_stores/bigquery.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/sdk/python/feast/infra/offline_stores/bigquery.py b/sdk/python/feast/infra/offline_stores/bigquery.py index 288b032f348..44e62d6ad1a 100644 --- a/sdk/python/feast/infra/offline_stores/bigquery.py +++ b/sdk/python/feast/infra/offline_stores/bigquery.py @@ -100,9 +100,7 @@ def pull_latest_from_table_or_query( if created_timestamp_column: timestamps.append(created_timestamp_column) timestamp_desc_string = " DESC, ".join(timestamps) + " DESC" - field_string = ", ".join( - join_key_columns + feature_name_columns + timestamps - ) + field_string = ", ".join(join_key_columns + feature_name_columns + timestamps) client = _get_bigquery_client( project=config.offline_store.project_id, From af2576ecc6ca749f9d7c622067d5964073064bdb Mon Sep 17 00:00:00 2001 From: pyalex Date: Thu, 27 Jan 2022 16:15:39 +0700 Subject: [PATCH 07/17] move ValidationReference proto message Signed-off-by: pyalex --- protos/feast/core/ValidationProfile.proto | 11 ++++++- protos/feast/core/ValidationReference.proto | 35 --------------------- 2 files changed, 10 insertions(+), 36 deletions(-) delete mode 100644 protos/feast/core/ValidationReference.proto diff --git a/protos/feast/core/ValidationProfile.proto b/protos/feast/core/ValidationProfile.proto index 4c67b263213..31c4e150a07 100644 --- a/protos/feast/core/ValidationProfile.proto +++ b/protos/feast/core/ValidationProfile.proto @@ -19,10 +19,11 @@ syntax = "proto3"; package feast.core; option java_package = "feast.proto.core"; -option java_outer_classname = "SavedDatasetProto"; +option java_outer_classname = "ValidationProfile"; option go_package = "github.com/feast-dev/feast/sdk/go/protos/feast/core"; import "google/protobuf/timestamp.proto"; +import "feast/core/SavedDataset.proto"; message GEValidationProfiler { message UserDefinedProfiler { @@ -37,3 +38,11 @@ message GEValidationProfile { // JSON-serialized ExpectationSuite object bytes expectation_suite = 1; } + +message ValidationReference { + SavedDataset dataset = 1; + + oneof profiler { + GEValidationProfiler ge_profiler = 2; + } +} diff --git a/protos/feast/core/ValidationReference.proto b/protos/feast/core/ValidationReference.proto deleted file mode 100644 index 56faddfbf1d..00000000000 --- a/protos/feast/core/ValidationReference.proto +++ /dev/null @@ -1,35 +0,0 @@ -// -// Copyright 2021 The Feast Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - - -syntax = "proto3"; - -package feast.core; -option java_package = "feast.proto.core"; -option java_outer_classname = "SavedDatasetProto"; -option go_package = "github.com/feast-dev/feast/sdk/go/protos/feast/core"; - -import "feast/core/SavedDataset.proto"; -import "feast/core/ValidationProfile.proto"; - - -message ValidationReference { - SavedDataset dataset = 1; - - oneof profiler { - GEValidationProfiler ge_profiler = 2; - } -} \ No newline at end of file From 17c4ec0f1e7db1a4b61edfb7a477251d36f96113 Mon Sep 17 00:00:00 2001 From: pyalex Date: Thu, 27 Jan 2022 18:26:36 +0700 Subject: [PATCH 08/17] grammar in docs Signed-off-by: pyalex --- docs/reference/dqm.md | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/docs/reference/dqm.md b/docs/reference/dqm.md index 3f49c06274a..087fad694c3 100644 --- a/docs/reference/dqm.md +++ b/docs/reference/dqm.md @@ -7,19 +7,19 @@ Validation could be applied during: * [planned] Reading features from an online store Its goal is to address several complex data problems, namely: -* Data Consistency - new training dataset could be significantly different from previous, which will require change in model architecture. -* Issues/bugs in upstream pipeline - bug in upstream could case invalid values to overwrite existing valid values in an online store. -* Training/serving skew - distribution shift could significantly decrease performance of the model. +* Data Consistency - new training dataset could be significantly different from the previous, which might require a change in model architecture. +* Issues/bugs in the upstream pipeline - bug in upstream could cause invalid values to overwrite existing valid values in an online store. +* Training/serving skew - distribution shift could significantly decrease the performance of the model. -> By “monitoring data quality” we understand verifying that the characteristics of tested dataset (we call it dataset's profile) are "equivalent" to the characteristics of reference dataset. -> Eg, data currently passed to the model hasn’t changed significantly since the model was trained and expectations implicitly made by ML algorithm during training are still met. +> By “monitoring data quality” we understand verifying that the characteristics of the tested dataset (we call this characteristics dataset's profile) are "equivalent" to the characteristics of the reference dataset. +> Eg, data currently passed to the model hasn’t changed significantly since the model was trained, and expectations implicitly made by ML algorithm during training are still met. > How exactly profiles equivalency should be measured is up to the user. ### Overview -Validation process consists of the next steps: +The validation process consists of the next steps: 1. User prepares reference dataset (currently only [saved dataset](../getting-started/concepts/dataset.md) from historical retrieval is supported). -2. User defines profiler function, which should produce profile by given dataset. +2. User defines profiler function, which should produce profile by given dataset (currently only profilers based on [Great Expectations](https://docs.greatexpectations.io) are allowed). 3. Validation of tested dataset is performed with reference dataset and profiler provided as parameters. ### Preparations @@ -30,7 +30,7 @@ pip install 'feast[dqm]' ### Dataset profile Currently, Feast supports only [great expectation's](https://greatexpectations.io/) [ExpectationSuite](https://legacy.docs.greatexpectations.io/en/latest/autoapi/great_expectations/core/expectation_suite/index.html#great_expectations.core.expectation_suite.ExpectationSuite) -as dataset's profile. Hence, user needs to define a function (profiler) that would receive a dataset and return [ExpectationSuite](https://legacy.docs.greatexpectations.io/en/latest/autoapi/great_expectations/core/expectation_suite/index.html#great_expectations.core.expectation_suite.ExpectationSuite). +as dataset's profiler. Hence, the user needs to define a function (profiler) that would receive a dataset and return [ExpectationSuite](https://legacy.docs.greatexpectations.io/en/latest/autoapi/great_expectations/core/expectation_suite/index.html#great_expectations.core.expectation_suite.ExpectationSuite). Either automatic profiler or user selected expectations could be used in profiler function: ```python @@ -60,10 +60,10 @@ def manual_profiler(dataset: Dataset) -> ExpectationSuite: ### Validating Training Dataset -During retrieval of historical features additional parameter `validation_reference` could be passed. -If this parameter is supplied `get_historical_features` will return `RetrievalJobWithValidation` instead of simple `RetrievalJob`. +During retrieval of historical features, additional parameter `validation_reference` could be passed. +If this parameter is supplied, `get_historical_features` will return instance `RetrievalJobWithValidation` instead of isntance of `RetrievalJob`. Such job will run validation once dataset is materialized (when `.to_df()` or `.to_arrow()` called). In case if validation successful materialized dataset is returned (no change to previous/regular behavior). -Otherwise `feast.dqm.errors.ValidationFailed` exception would be raised. It will consist of all details for expectations that didn't pass. +Otherwise, `feast.dqm.errors.ValidationFailed` exception would be raised. It will consist of all details for expectations that didn't pass. ```python from feast import FeatureStore From d3449db3f41d50c6ecb7a0e2bc7e932b9f404380 Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Tue, 1 Feb 2022 04:28:35 +0200 Subject: [PATCH 09/17] Update docs/reference/dqm.md Co-authored-by: Danny Chiao Signed-off-by: pyalex --- docs/reference/dqm.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/reference/dqm.md b/docs/reference/dqm.md index 087fad694c3..d722f5ac31e 100644 --- a/docs/reference/dqm.md +++ b/docs/reference/dqm.md @@ -7,13 +7,13 @@ Validation could be applied during: * [planned] Reading features from an online store Its goal is to address several complex data problems, namely: -* Data Consistency - new training dataset could be significantly different from the previous, which might require a change in model architecture. -* Issues/bugs in the upstream pipeline - bug in upstream could cause invalid values to overwrite existing valid values in an online store. +* Data consistency - new training datasets can be significantly different from previous datasets. This might require a change in model architecture. +* Issues/bugs in the upstream pipeline - bugs in upstream pipelines can cause invalid values to overwrite existing valid values in an online store. * Training/serving skew - distribution shift could significantly decrease the performance of the model. -> By “monitoring data quality” we understand verifying that the characteristics of the tested dataset (we call this characteristics dataset's profile) are "equivalent" to the characteristics of the reference dataset. +> To monitor data quality, we check that the characteristics of the tested dataset (aka the tested dataset's profile) are "equivalent" to the characteristics of the reference dataset. > Eg, data currently passed to the model hasn’t changed significantly since the model was trained, and expectations implicitly made by ML algorithm during training are still met. -> How exactly profiles equivalency should be measured is up to the user. +> How exactly profile equivalency should be measured is up to the user. ### Overview From 1ca7eaadccd8e99514351a58111006070adbf700 Mon Sep 17 00:00:00 2001 From: pyalex Date: Tue, 1 Feb 2022 10:51:39 +0800 Subject: [PATCH 10/17] rebase & resolve conflicts Signed-off-by: pyalex --- docs/reference/dqm.md | 1 - .../requirements/py3.7-ci-requirements.txt | 252 +++++++++++++-- .../requirements/py3.8-ci-requirements.txt | 287 +++++++++++++++-- .../requirements/py3.9-ci-requirements.txt | 289 ++++++++++++++++-- sdk/python/setup.py | 18 +- 5 files changed, 742 insertions(+), 105 deletions(-) diff --git a/docs/reference/dqm.md b/docs/reference/dqm.md index d722f5ac31e..bc760928332 100644 --- a/docs/reference/dqm.md +++ b/docs/reference/dqm.md @@ -12,7 +12,6 @@ Its goal is to address several complex data problems, namely: * Training/serving skew - distribution shift could significantly decrease the performance of the model. > To monitor data quality, we check that the characteristics of the tested dataset (aka the tested dataset's profile) are "equivalent" to the characteristics of the reference dataset. -> Eg, data currently passed to the model hasn’t changed significantly since the model was trained, and expectations implicitly made by ML algorithm during training are still met. > How exactly profile equivalency should be measured is up to the user. ### Overview diff --git a/sdk/python/requirements/py3.7-ci-requirements.txt b/sdk/python/requirements/py3.7-ci-requirements.txt index 293b44e0531..d5f654e515c 100644 --- a/sdk/python/requirements/py3.7-ci-requirements.txt +++ b/sdk/python/requirements/py3.7-ci-requirements.txt @@ -20,11 +20,21 @@ aiosignal==1.2.0 # via aiohttp alabaster==0.7.12 # via sphinx +altair==4.2.0 + # via great-expectations anyio==3.5.0 # via starlette appdirs==1.4.4 # via black -asgiref==3.4.1 +appnope==0.1.2 + # via + # ipykernel + # ipython +argon2-cffi==21.3.0 + # via notebook +argon2-cffi-bindings==21.2.0 + # via argon2-cffi +asgiref==3.5.0 # via uvicorn asn1crypto==1.4.0 # via @@ -57,13 +67,21 @@ azure-storage-blob==12.9.0 # via adlfs babel==2.9.1 # via sphinx +backcall==0.2.0 + # via ipython +backports.zoneinfo==0.2.1 + # via + # pytz-deprecation-shim + # tzlocal black==19.10b0 # via feast (setup.py) -boto3==1.20.40 +bleach==4.1.0 + # via nbconvert +boto3==1.20.46 # via # feast (setup.py) # moto -botocore==1.23.40 +botocore==1.23.46 # via # boto3 # moto @@ -80,12 +98,13 @@ certifi==2021.10.8 # snowflake-connector-python cffi==1.15.0 # via + # argon2-cffi-bindings # azure-datalake-store # cryptography # snowflake-connector-python cfgv==3.3.1 # via pre-commit -charset-normalizer==2.0.10 +charset-normalizer==2.0.11 # via # aiohttp # requests @@ -94,11 +113,12 @@ click==8.0.3 # via # black # feast (setup.py) + # great-expectations # pip-tools # uvicorn colorama==0.4.4 # via feast (setup.py) -coverage[toml]==6.2 +coverage[toml]==6.3 # via pytest-cov cryptography==3.3.2 # via @@ -108,10 +128,17 @@ cryptography==3.3.2 # feast (setup.py) # moto # msal + # pyjwt # pyopenssl # snowflake-connector-python +debugpy==1.5.1 + # via ipykernel decorator==5.1.1 - # via gcsfs + # via + # gcsfs + # ipython +defusedxml==0.7.1 + # via nbconvert deprecated==1.2.13 # via redis deprecation==2.1.0 @@ -128,9 +155,14 @@ docutils==0.17.1 # via # sphinx # sphinx-rtd-theme +entrypoints==0.3 + # via + # altair + # jupyter-client + # nbconvert execnet==1.9.0 # via pytest-xdist -fastapi==0.72.0 +fastapi==0.73.0 # via feast (setup.py) fastavro==1.4.9 # via @@ -208,6 +240,8 @@ googleapis-common-protos==1.52.0 # feast (setup.py) # google-api-core # tensorflow-metadata +great-expectations==0.14.4 + # via feast (setup.py) grpcio==1.43.0 # via # feast (setup.py) @@ -232,7 +266,7 @@ httplib2==0.20.2 # google-auth-httplib2 httptools==0.3.0 # via uvicorn -identify==2.4.4 +identify==2.4.7 # via pre-commit idna==3.3 # via @@ -246,6 +280,7 @@ importlib-metadata==4.2.0 # via # click # flake8 + # great-expectations # jsonschema # moto # pep517 @@ -258,22 +293,66 @@ importlib-resources==5.4.0 # via jsonschema iniconfig==1.1.1 # via pytest +ipykernel==6.7.0 + # via + # ipywidgets + # notebook +ipython==7.31.1 + # via + # ipykernel + # ipywidgets +ipython-genutils==0.2.0 + # via + # ipywidgets + # nbformat + # notebook +ipywidgets==7.6.5 + # via great-expectations isodate==0.6.1 # via msrest isort==5.10.1 # via feast (setup.py) +jedi==0.18.1 + # via ipython jinja2==3.0.3 # via + # altair # feast (setup.py) + # great-expectations # moto + # nbconvert + # notebook # sphinx jmespath==0.10.0 # via # boto3 # botocore +jsonpatch==1.32 + # via great-expectations +jsonpointer==2.2 + # via jsonpatch jsonschema==4.4.0 - # via feast (setup.py) -libcst==0.4.0 + # via + # altair + # feast (setup.py) + # great-expectations + # nbformat +jupyter-client==7.1.2 + # via + # ipykernel + # nbclient + # notebook +jupyter-core==4.9.1 + # via + # jupyter-client + # nbconvert + # nbformat + # notebook +jupyterlab-pygments==0.1.2 + # via nbconvert +jupyterlab-widgets==1.0.2 + # via ipywidgets +libcst==0.4.1 # via # google-cloud-bigquery-storage # google-cloud-datastore @@ -281,15 +360,23 @@ markupsafe==2.0.1 # via # jinja2 # moto +matplotlib-inline==0.1.3 + # via + # ipykernel + # ipython mccabe==0.6.1 # via flake8 minio==7.1.0 # via feast (setup.py) +mistune==0.8.4 + # via + # great-expectations + # nbconvert mmh3==3.0.0 # via feast (setup.py) mock==2.0.0 # via feast (setup.py) -moto==3.0.0 +moto==3.0.2 # via feast (setup.py) msal==1.16.0 # via @@ -305,7 +392,7 @@ msrest==0.6.21 # msrestazure msrestazure==0.6.4 # via adlfs -multidict==5.2.0 +multidict==6.0.2 # via # aiohttp # yarl @@ -317,19 +404,41 @@ mypy-extensions==0.4.3 # typing-inspect mypy-protobuf==3.1.0 # via feast (setup.py) +nbclient==0.5.10 + # via nbconvert +nbconvert==6.4.1 + # via notebook +nbformat==5.1.3 + # via + # ipywidgets + # nbclient + # nbconvert + # notebook +nest-asyncio==1.5.4 + # via + # ipykernel + # jupyter-client + # nbclient + # notebook nodeenv==1.6.0 # via pre-commit +notebook==6.4.8 + # via widgetsnbextension numpy==1.21.5 # via + # altair + # great-expectations # pandas # pandavro # pyarrow -oauthlib==3.1.1 + # scipy +oauthlib==3.2.0 # via requests-oauthlib oscrypto==1.2.1 # via snowflake-connector-python packaging==21.3 # via + # bleach # deprecation # google-api-core # google-cloud-bigquery @@ -339,17 +448,27 @@ packaging==21.3 # sphinx pandas==1.3.5 # via + # altair # feast (setup.py) + # great-expectations # pandavro # snowflake-connector-python pandavro==1.5.2 # via feast (setup.py) +pandocfilters==1.5.0 + # via nbconvert +parso==0.8.3 + # via jedi pathspec==0.9.0 # via black pbr==5.8.0 # via mock pep517==0.12.0 # via pip-tools +pexpect==4.8.0 + # via ipython +pickleshare==0.7.5 + # via ipython pip-tools==6.4.0 # via feast (setup.py) platformdirs==2.4.1 @@ -360,6 +479,10 @@ portalocker==2.3.2 # via msal-extensions pre-commit==2.17.0 # via feast (setup.py) +prometheus-client==0.13.1 + # via notebook +prompt-toolkit==3.0.26 + # via ipython proto-plus==1.19.6 # via # feast (setup.py) @@ -367,7 +490,7 @@ proto-plus==1.19.6 # google-cloud-bigquery-storage # google-cloud-datastore # google-cloud-firestore -protobuf==3.19.3 +protobuf==3.19.4 # via # feast (setup.py) # google-api-core @@ -379,6 +502,10 @@ protobuf==3.19.3 # mypy-protobuf # proto-plus # tensorflow-metadata +ptyprocess==0.7.0 + # via + # pexpect + # terminado py==1.11.0 # via # pytest @@ -399,7 +526,7 @@ pycodestyle==2.8.0 # via flake8 pycparser==2.21 # via cffi -pycryptodomex==3.13.0 +pycryptodomex==3.14.0 # via snowflake-connector-python pydantic==1.9.0 # via @@ -408,7 +535,11 @@ pydantic==1.9.0 pyflakes==2.4.0 # via flake8 pygments==2.11.2 - # via sphinx + # via + # ipython + # jupyterlab-pygments + # nbconvert + # sphinx pyjwt[crypto]==2.3.0 # via # adal @@ -416,8 +547,9 @@ pyjwt[crypto]==2.3.0 # snowflake-connector-python pyopenssl==21.0.0 # via snowflake-connector-python -pyparsing==3.0.7 +pyparsing==2.4.7 # via + # great-expectations # httplib2 # packaging pyrsistent==0.18.1 @@ -454,6 +586,8 @@ python-dateutil==2.8.2 # adal # botocore # google-cloud-bigquery + # great-expectations + # jupyter-client # moto # pandas python-dotenv==0.19.2 @@ -462,16 +596,23 @@ pytz==2021.3 # via # babel # google-api-core + # great-expectations # moto # pandas # snowflake-connector-python +pytz-deprecation-shim==0.1.0.post0 + # via tzlocal pyyaml==6.0 # via # feast (setup.py) # libcst # pre-commit # uvicorn -redis==4.1.1 +pyzmq==22.3.0 + # via + # jupyter-client + # notebook +redis==4.1.2 # via feast (setup.py) regex==2022.1.18 # via black @@ -487,6 +628,7 @@ requests==2.27.1 # google-api-core # google-cloud-bigquery # google-cloud-storage + # great-expectations # moto # msal # msrest @@ -494,7 +636,7 @@ requests==2.27.1 # responses # snowflake-connector-python # sphinx -requests-oauthlib==1.3.0 +requests-oauthlib==1.3.1 # via # google-auth-oauthlib # msrest @@ -502,13 +644,22 @@ responses==0.17.0 # via moto rsa==4.8 # via google-auth +ruamel.yaml==0.17.17 + # via great-expectations +ruamel.yaml.clib==0.2.6 + # via ruamel.yaml s3transfer==0.5.0 # via boto3 +scipy==1.7.3 + # via great-expectations +send2trash==1.8.0 + # via notebook six==1.16.0 # via # absl-py # azure-core # azure-identity + # bleach # cryptography # google-api-core # google-auth @@ -516,6 +667,7 @@ six==1.16.0 # google-cloud-core # google-resumable-media # grpcio + # isodate # mock # msrestazure # pandavro @@ -555,8 +707,14 @@ tenacity==8.0.1 # via feast (setup.py) tensorflow-metadata==1.6.0 # via feast (setup.py) +termcolor==1.1.0 + # via great-expectations +terminado==0.13.1 + # via notebook testcontainers==3.4.2 # via feast (setup.py) +testpath==0.5.0 + # via nbconvert toml==0.10.2 # via # black @@ -568,40 +726,64 @@ tomli==2.0.0 # coverage # mypy # pep517 +toolz==0.11.2 + # via altair +tornado==6.1 + # via + # ipykernel + # jupyter-client + # notebook + # terminado tqdm==4.62.3 - # via feast (setup.py) -typed-ast==1.5.1 + # via + # feast (setup.py) + # great-expectations +traitlets==5.1.1 + # via + # ipykernel + # ipython + # ipywidgets + # jupyter-client + # jupyter-core + # matplotlib-inline + # nbclient + # nbconvert + # nbformat + # notebook +typed-ast==1.5.2 # via # black # mypy -types-futures==3.3.7 +types-futures==3.3.8 # via types-protobuf -types-protobuf==3.19.5 +types-protobuf==3.19.7 # via # feast (setup.py) # mypy-protobuf -types-python-dateutil==2.8.8 +types-python-dateutil==2.8.9 # via feast (setup.py) types-pytz==2021.3.4 # via feast (setup.py) -types-pyyaml==6.0.3 +types-pyyaml==6.0.4 # via feast (setup.py) -types-redis==4.1.10 +types-redis==4.1.13 # via feast (setup.py) -types-requests==2.27.7 +types-requests==2.27.8 # via feast (setup.py) -types-setuptools==57.4.7 +types-setuptools==57.4.8 # via feast (setup.py) types-tabulate==0.8.5 # via feast (setup.py) -types-urllib3==1.26.7 +types-urllib3==1.26.8 # via types-requests typing-extensions==4.0.1 # via # aiohttp # anyio + # argon2-cffi # asgiref # async-timeout + # great-expectations # h11 # importlib-metadata # jsonschema @@ -614,6 +796,10 @@ typing-extensions==4.0.1 # yarl typing-inspect==0.7.1 # via libcst +tzdata==2021.5 + # via pytz-deprecation-shim +tzlocal==4.1 + # via great-expectations uritemplate==4.1.1 # via google-api-python-client urllib3==1.26.8 @@ -623,7 +809,7 @@ urllib3==1.26.8 # minio # requests # responses -uvicorn[standard]==0.17.0 +uvicorn[standard]==0.17.1 # via feast (setup.py) uvloop==0.16.0 # via uvicorn @@ -631,6 +817,10 @@ virtualenv==20.13.0 # via pre-commit watchgod==0.7 # via uvicorn +wcwidth==0.2.5 + # via prompt-toolkit +webencodings==0.5.1 + # via bleach websocket-client==1.2.3 # via docker websockets==10.1 @@ -639,6 +829,8 @@ werkzeug==2.0.2 # via moto wheel==0.37.1 # via pip-tools +widgetsnbextension==3.5.2 + # via ipywidgets wrapt==1.13.3 # via # deprecated diff --git a/sdk/python/requirements/py3.8-ci-requirements.txt b/sdk/python/requirements/py3.8-ci-requirements.txt index 3cdc118144f..7a94294c956 100644 --- a/sdk/python/requirements/py3.8-ci-requirements.txt +++ b/sdk/python/requirements/py3.8-ci-requirements.txt @@ -20,11 +20,21 @@ aiosignal==1.2.0 # via aiohttp alabaster==0.7.12 # via sphinx +altair==4.2.0 + # via great-expectations anyio==3.5.0 # via starlette appdirs==1.4.4 # via black -asgiref==3.4.1 +appnope==0.1.2 + # via + # ipykernel + # ipython +argon2-cffi==21.3.0 + # via notebook +argon2-cffi-bindings==21.2.0 + # via argon2-cffi +asgiref==3.5.0 # via uvicorn asn1crypto==1.4.0 # via @@ -34,6 +44,8 @@ assertpy==1.1 # via feast (setup.py) async-timeout==4.0.2 # via aiohttp +asynctest==0.13.0 + # via aiohttp attrs==21.4.0 # via # aiohttp @@ -55,13 +67,21 @@ azure-storage-blob==12.9.0 # via adlfs babel==2.9.1 # via sphinx +backcall==0.2.0 + # via ipython +backports.zoneinfo==0.2.1 + # via + # pytz-deprecation-shim + # tzlocal black==19.10b0 # via feast (setup.py) -boto3==1.20.40 +bleach==4.1.0 + # via nbconvert +boto3==1.20.46 # via # feast (setup.py) # moto -botocore==1.23.40 +botocore==1.23.46 # via # boto3 # moto @@ -78,12 +98,13 @@ certifi==2021.10.8 # snowflake-connector-python cffi==1.15.0 # via + # argon2-cffi-bindings # azure-datalake-store # cryptography # snowflake-connector-python cfgv==3.3.1 # via pre-commit -charset-normalizer==2.0.10 +charset-normalizer==2.0.11 # via # aiohttp # requests @@ -92,11 +113,12 @@ click==8.0.3 # via # black # feast (setup.py) + # great-expectations # pip-tools # uvicorn colorama==0.4.4 # via feast (setup.py) -coverage[toml]==6.2 +coverage[toml]==6.3 # via pytest-cov cryptography==3.3.2 # via @@ -106,10 +128,17 @@ cryptography==3.3.2 # feast (setup.py) # moto # msal + # pyjwt # pyopenssl # snowflake-connector-python +debugpy==1.5.1 + # via ipykernel decorator==5.1.1 - # via gcsfs + # via + # gcsfs + # ipython +defusedxml==0.7.1 + # via nbconvert deprecated==1.2.13 # via redis deprecation==2.1.0 @@ -126,9 +155,14 @@ docutils==0.17.1 # via # sphinx # sphinx-rtd-theme +entrypoints==0.3 + # via + # altair + # jupyter-client + # nbconvert execnet==1.9.0 # via pytest-xdist -fastapi==0.72.0 +fastapi==0.73.0 # via feast (setup.py) fastavro==1.4.9 # via @@ -206,6 +240,8 @@ googleapis-common-protos==1.52.0 # feast (setup.py) # google-api-core # tensorflow-metadata +great-expectations==0.14.4 + # via feast (setup.py) grpcio==1.43.0 # via # feast (setup.py) @@ -230,7 +266,7 @@ httplib2==0.20.2 # google-auth-httplib2 httptools==0.3.0 # via uvicorn -identify==2.4.4 +identify==2.4.7 # via pre-commit idna==3.3 # via @@ -240,26 +276,83 @@ idna==3.3 # yarl imagesize==1.3.0 # via sphinx +importlib-metadata==4.2.0 + # via + # click + # flake8 + # great-expectations + # jsonschema + # moto + # pep517 + # pluggy + # pre-commit + # pytest + # redis + # virtualenv importlib-resources==5.4.0 # via jsonschema iniconfig==1.1.1 # via pytest +ipykernel==6.7.0 + # via + # ipywidgets + # notebook +ipython==7.31.1 + # via + # ipykernel + # ipywidgets +ipython-genutils==0.2.0 + # via + # ipywidgets + # nbformat + # notebook +ipywidgets==7.6.5 + # via great-expectations isodate==0.6.1 # via msrest isort==5.10.1 # via feast (setup.py) +jedi==0.18.1 + # via ipython jinja2==3.0.3 # via + # altair # feast (setup.py) + # great-expectations # moto + # nbconvert + # notebook # sphinx jmespath==0.10.0 # via # boto3 # botocore +jsonpatch==1.32 + # via great-expectations +jsonpointer==2.2 + # via jsonpatch jsonschema==4.4.0 - # via feast (setup.py) -libcst==0.4.0 + # via + # altair + # feast (setup.py) + # great-expectations + # nbformat +jupyter-client==7.1.2 + # via + # ipykernel + # nbclient + # notebook +jupyter-core==4.9.1 + # via + # jupyter-client + # nbconvert + # nbformat + # notebook +jupyterlab-pygments==0.1.2 + # via nbconvert +jupyterlab-widgets==1.0.2 + # via ipywidgets +libcst==0.4.1 # via # google-cloud-bigquery-storage # google-cloud-datastore @@ -267,15 +360,23 @@ markupsafe==2.0.1 # via # jinja2 # moto +matplotlib-inline==0.1.3 + # via + # ipykernel + # ipython mccabe==0.6.1 # via flake8 minio==7.1.0 # via feast (setup.py) +mistune==0.8.4 + # via + # great-expectations + # nbconvert mmh3==3.0.0 # via feast (setup.py) mock==2.0.0 # via feast (setup.py) -moto==3.0.0 +moto==3.0.2 # via feast (setup.py) msal==1.16.0 # via @@ -291,7 +392,7 @@ msrest==0.6.21 # msrestazure msrestazure==0.6.4 # via adlfs -multidict==5.2.0 +multidict==6.0.2 # via # aiohttp # yarl @@ -303,19 +404,41 @@ mypy-extensions==0.4.3 # typing-inspect mypy-protobuf==3.1.0 # via feast (setup.py) +nbclient==0.5.10 + # via nbconvert +nbconvert==6.4.1 + # via notebook +nbformat==5.1.3 + # via + # ipywidgets + # nbclient + # nbconvert + # notebook +nest-asyncio==1.5.4 + # via + # ipykernel + # jupyter-client + # nbclient + # notebook nodeenv==1.6.0 # via pre-commit -numpy==1.22.1 +notebook==6.4.8 + # via widgetsnbextension +numpy==1.21.5 # via + # altair + # great-expectations # pandas # pandavro # pyarrow -oauthlib==3.1.1 + # scipy +oauthlib==3.2.0 # via requests-oauthlib oscrypto==1.2.1 # via snowflake-connector-python packaging==21.3 # via + # bleach # deprecation # google-api-core # google-cloud-bigquery @@ -325,17 +448,27 @@ packaging==21.3 # sphinx pandas==1.3.5 # via + # altair # feast (setup.py) + # great-expectations # pandavro # snowflake-connector-python pandavro==1.5.2 # via feast (setup.py) +pandocfilters==1.5.0 + # via nbconvert +parso==0.8.3 + # via jedi pathspec==0.9.0 # via black pbr==5.8.0 # via mock pep517==0.12.0 # via pip-tools +pexpect==4.8.0 + # via ipython +pickleshare==0.7.5 + # via ipython pip-tools==6.4.0 # via feast (setup.py) platformdirs==2.4.1 @@ -346,6 +479,10 @@ portalocker==2.3.2 # via msal-extensions pre-commit==2.17.0 # via feast (setup.py) +prometheus-client==0.13.1 + # via notebook +prompt-toolkit==3.0.26 + # via ipython proto-plus==1.19.6 # via # feast (setup.py) @@ -353,7 +490,7 @@ proto-plus==1.19.6 # google-cloud-bigquery-storage # google-cloud-datastore # google-cloud-firestore -protobuf==3.19.3 +protobuf==3.19.4 # via # feast (setup.py) # google-api-core @@ -365,6 +502,10 @@ protobuf==3.19.3 # mypy-protobuf # proto-plus # tensorflow-metadata +ptyprocess==0.7.0 + # via + # pexpect + # terminado py==1.11.0 # via # pytest @@ -385,7 +526,7 @@ pycodestyle==2.8.0 # via flake8 pycparser==2.21 # via cffi -pycryptodomex==3.13.0 +pycryptodomex==3.14.0 # via snowflake-connector-python pydantic==1.9.0 # via @@ -394,7 +535,11 @@ pydantic==1.9.0 pyflakes==2.4.0 # via flake8 pygments==2.11.2 - # via sphinx + # via + # ipython + # jupyterlab-pygments + # nbconvert + # sphinx pyjwt[crypto]==2.3.0 # via # adal @@ -402,8 +547,9 @@ pyjwt[crypto]==2.3.0 # snowflake-connector-python pyopenssl==21.0.0 # via snowflake-connector-python -pyparsing==3.0.7 +pyparsing==2.4.7 # via + # great-expectations # httplib2 # packaging pyrsistent==0.18.1 @@ -440,6 +586,8 @@ python-dateutil==2.8.2 # adal # botocore # google-cloud-bigquery + # great-expectations + # jupyter-client # moto # pandas python-dotenv==0.19.2 @@ -448,16 +596,23 @@ pytz==2021.3 # via # babel # google-api-core + # great-expectations # moto # pandas # snowflake-connector-python +pytz-deprecation-shim==0.1.0.post0 + # via tzlocal pyyaml==6.0 # via # feast (setup.py) # libcst # pre-commit # uvicorn -redis==4.1.1 +pyzmq==22.3.0 + # via + # jupyter-client + # notebook +redis==4.1.2 # via feast (setup.py) regex==2022.1.18 # via black @@ -473,6 +628,7 @@ requests==2.27.1 # google-api-core # google-cloud-bigquery # google-cloud-storage + # great-expectations # moto # msal # msrest @@ -480,7 +636,7 @@ requests==2.27.1 # responses # snowflake-connector-python # sphinx -requests-oauthlib==1.3.0 +requests-oauthlib==1.3.1 # via # google-auth-oauthlib # msrest @@ -488,13 +644,22 @@ responses==0.17.0 # via moto rsa==4.8 # via google-auth +ruamel.yaml==0.17.17 + # via great-expectations +ruamel.yaml.clib==0.2.6 + # via ruamel.yaml s3transfer==0.5.0 # via boto3 +scipy==1.7.3 + # via great-expectations +send2trash==1.8.0 + # via notebook six==1.16.0 # via # absl-py # azure-core # azure-identity + # bleach # cryptography # google-api-core # google-auth @@ -502,6 +667,7 @@ six==1.16.0 # google-cloud-core # google-resumable-media # grpcio + # isodate # mock # msrestazure # pandavro @@ -541,8 +707,14 @@ tenacity==8.0.1 # via feast (setup.py) tensorflow-metadata==1.6.0 # via feast (setup.py) +termcolor==1.1.0 + # via great-expectations +terminado==0.13.1 + # via notebook testcontainers==3.4.2 # via feast (setup.py) +testpath==0.5.0 + # via nbconvert toml==0.10.2 # via # black @@ -554,40 +726,80 @@ tomli==2.0.0 # coverage # mypy # pep517 +toolz==0.11.2 + # via altair +tornado==6.1 + # via + # ipykernel + # jupyter-client + # notebook + # terminado tqdm==4.62.3 - # via feast (setup.py) -typed-ast==1.5.1 - # via black -types-futures==3.3.7 + # via + # feast (setup.py) + # great-expectations +traitlets==5.1.1 + # via + # ipykernel + # ipython + # ipywidgets + # jupyter-client + # jupyter-core + # matplotlib-inline + # nbclient + # nbconvert + # nbformat + # notebook +typed-ast==1.5.2 + # via + # black + # mypy +types-futures==3.3.8 # via types-protobuf -types-protobuf==3.19.5 +types-protobuf==3.19.7 # via # feast (setup.py) # mypy-protobuf -types-python-dateutil==2.8.8 +types-python-dateutil==2.8.9 # via feast (setup.py) types-pytz==2021.3.4 # via feast (setup.py) -types-pyyaml==6.0.3 +types-pyyaml==6.0.4 # via feast (setup.py) -types-redis==4.1.10 +types-redis==4.1.13 # via feast (setup.py) -types-requests==2.27.7 +types-requests==2.27.8 # via feast (setup.py) -types-setuptools==57.4.7 +types-setuptools==57.4.8 # via feast (setup.py) types-tabulate==0.8.5 # via feast (setup.py) -types-urllib3==1.26.7 +types-urllib3==1.26.8 # via types-requests typing-extensions==4.0.1 # via + # aiohttp + # anyio + # argon2-cffi + # asgiref + # async-timeout + # great-expectations + # h11 + # importlib-metadata + # jsonschema # libcst # mypy # pydantic + # starlette # typing-inspect + # uvicorn + # yarl typing-inspect==0.7.1 # via libcst +tzdata==2021.5 + # via pytz-deprecation-shim +tzlocal==4.1 + # via great-expectations uritemplate==4.1.1 # via google-api-python-client urllib3==1.26.8 @@ -597,7 +809,7 @@ urllib3==1.26.8 # minio # requests # responses -uvicorn[standard]==0.17.0 +uvicorn[standard]==0.17.1 # via feast (setup.py) uvloop==0.16.0 # via uvicorn @@ -605,6 +817,10 @@ virtualenv==20.13.0 # via pre-commit watchgod==0.7 # via uvicorn +wcwidth==0.2.5 + # via prompt-toolkit +webencodings==0.5.1 + # via bleach websocket-client==1.2.3 # via docker websockets==10.1 @@ -613,6 +829,8 @@ werkzeug==2.0.2 # via moto wheel==0.37.1 # via pip-tools +widgetsnbextension==3.5.2 + # via ipywidgets wrapt==1.13.3 # via # deprecated @@ -622,7 +840,10 @@ xmltodict==0.12.0 yarl==1.7.2 # via aiohttp zipp==3.7.0 - # via importlib-resources + # via + # importlib-metadata + # importlib-resources + # pep517 # The following packages are considered to be unsafe in a requirements file: # pip diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index 69247a2c7dd..1421d7e3c31 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -20,11 +20,21 @@ aiosignal==1.2.0 # via aiohttp alabaster==0.7.12 # via sphinx +altair==4.2.0 + # via great-expectations anyio==3.5.0 # via starlette appdirs==1.4.4 # via black -asgiref==3.4.1 +appnope==0.1.2 + # via + # ipykernel + # ipython +argon2-cffi==21.3.0 + # via notebook +argon2-cffi-bindings==21.2.0 + # via argon2-cffi +asgiref==3.5.0 # via uvicorn asn1crypto==1.4.0 # via @@ -34,6 +44,8 @@ assertpy==1.1 # via feast (setup.py) async-timeout==4.0.2 # via aiohttp +asynctest==0.13.0 + # via aiohttp attrs==21.4.0 # via # aiohttp @@ -55,13 +67,21 @@ azure-storage-blob==12.9.0 # via adlfs babel==2.9.1 # via sphinx +backcall==0.2.0 + # via ipython +backports.zoneinfo==0.2.1 + # via + # pytz-deprecation-shim + # tzlocal black==19.10b0 # via feast (setup.py) -boto3==1.20.40 +bleach==4.1.0 + # via nbconvert +boto3==1.20.46 # via # feast (setup.py) # moto -botocore==1.23.40 +botocore==1.23.46 # via # boto3 # moto @@ -78,12 +98,13 @@ certifi==2021.10.8 # snowflake-connector-python cffi==1.15.0 # via + # argon2-cffi-bindings # azure-datalake-store # cryptography # snowflake-connector-python cfgv==3.3.1 # via pre-commit -charset-normalizer==2.0.10 +charset-normalizer==2.0.11 # via # aiohttp # requests @@ -92,11 +113,12 @@ click==8.0.3 # via # black # feast (setup.py) + # great-expectations # pip-tools # uvicorn colorama==0.4.4 # via feast (setup.py) -coverage[toml]==6.2 +coverage[toml]==6.3 # via pytest-cov cryptography==3.3.2 # via @@ -106,10 +128,17 @@ cryptography==3.3.2 # feast (setup.py) # moto # msal + # pyjwt # pyopenssl # snowflake-connector-python +debugpy==1.5.1 + # via ipykernel decorator==5.1.1 - # via gcsfs + # via + # gcsfs + # ipython +defusedxml==0.7.1 + # via nbconvert deprecated==1.2.13 # via redis deprecation==2.1.0 @@ -126,9 +155,14 @@ docutils==0.17.1 # via # sphinx # sphinx-rtd-theme +entrypoints==0.3 + # via + # altair + # jupyter-client + # nbconvert execnet==1.9.0 # via pytest-xdist -fastapi==0.72.0 +fastapi==0.73.0 # via feast (setup.py) fastavro==1.4.9 # via @@ -206,6 +240,8 @@ googleapis-common-protos==1.52.0 # feast (setup.py) # google-api-core # tensorflow-metadata +great-expectations==0.14.4 + # via feast (setup.py) grpcio==1.43.0 # via # feast (setup.py) @@ -230,7 +266,7 @@ httplib2==0.20.2 # google-auth-httplib2 httptools==0.3.0 # via uvicorn -identify==2.4.4 +identify==2.4.7 # via pre-commit idna==3.3 # via @@ -240,24 +276,83 @@ idna==3.3 # yarl imagesize==1.3.0 # via sphinx +importlib-metadata==4.2.0 + # via + # click + # flake8 + # great-expectations + # jsonschema + # moto + # pep517 + # pluggy + # pre-commit + # pytest + # redis + # virtualenv +importlib-resources==5.4.0 + # via jsonschema iniconfig==1.1.1 # via pytest +ipykernel==6.7.0 + # via + # ipywidgets + # notebook +ipython==7.31.1 + # via + # ipykernel + # ipywidgets +ipython-genutils==0.2.0 + # via + # ipywidgets + # nbformat + # notebook +ipywidgets==7.6.5 + # via great-expectations isodate==0.6.1 # via msrest isort==5.10.1 # via feast (setup.py) +jedi==0.18.1 + # via ipython jinja2==3.0.3 # via + # altair # feast (setup.py) + # great-expectations # moto + # nbconvert + # notebook # sphinx jmespath==0.10.0 # via # boto3 # botocore +jsonpatch==1.32 + # via great-expectations +jsonpointer==2.2 + # via jsonpatch jsonschema==4.4.0 - # via feast (setup.py) -libcst==0.4.0 + # via + # altair + # feast (setup.py) + # great-expectations + # nbformat +jupyter-client==7.1.2 + # via + # ipykernel + # nbclient + # notebook +jupyter-core==4.9.1 + # via + # jupyter-client + # nbconvert + # nbformat + # notebook +jupyterlab-pygments==0.1.2 + # via nbconvert +jupyterlab-widgets==1.0.2 + # via ipywidgets +libcst==0.4.1 # via # google-cloud-bigquery-storage # google-cloud-datastore @@ -265,15 +360,23 @@ markupsafe==2.0.1 # via # jinja2 # moto +matplotlib-inline==0.1.3 + # via + # ipykernel + # ipython mccabe==0.6.1 # via flake8 minio==7.1.0 # via feast (setup.py) +mistune==0.8.4 + # via + # great-expectations + # nbconvert mmh3==3.0.0 # via feast (setup.py) mock==2.0.0 # via feast (setup.py) -moto==3.0.0 +moto==3.0.2 # via feast (setup.py) msal==1.16.0 # via @@ -289,7 +392,7 @@ msrest==0.6.21 # msrestazure msrestazure==0.6.4 # via adlfs -multidict==5.2.0 +multidict==6.0.2 # via # aiohttp # yarl @@ -301,19 +404,41 @@ mypy-extensions==0.4.3 # typing-inspect mypy-protobuf==3.1.0 # via feast (setup.py) +nbclient==0.5.10 + # via nbconvert +nbconvert==6.4.1 + # via notebook +nbformat==5.1.3 + # via + # ipywidgets + # nbclient + # nbconvert + # notebook +nest-asyncio==1.5.4 + # via + # ipykernel + # jupyter-client + # nbclient + # notebook nodeenv==1.6.0 # via pre-commit -numpy==1.22.1 +notebook==6.4.8 + # via widgetsnbextension +numpy==1.21.5 # via + # altair + # great-expectations # pandas # pandavro # pyarrow -oauthlib==3.1.1 + # scipy +oauthlib==3.2.0 # via requests-oauthlib oscrypto==1.2.1 # via snowflake-connector-python packaging==21.3 # via + # bleach # deprecation # google-api-core # google-cloud-bigquery @@ -323,17 +448,27 @@ packaging==21.3 # sphinx pandas==1.3.5 # via + # altair # feast (setup.py) + # great-expectations # pandavro # snowflake-connector-python pandavro==1.5.2 # via feast (setup.py) +pandocfilters==1.5.0 + # via nbconvert +parso==0.8.3 + # via jedi pathspec==0.9.0 # via black pbr==5.8.0 # via mock pep517==0.12.0 # via pip-tools +pexpect==4.8.0 + # via ipython +pickleshare==0.7.5 + # via ipython pip-tools==6.4.0 # via feast (setup.py) platformdirs==2.4.1 @@ -344,6 +479,10 @@ portalocker==2.3.2 # via msal-extensions pre-commit==2.17.0 # via feast (setup.py) +prometheus-client==0.13.1 + # via notebook +prompt-toolkit==3.0.26 + # via ipython proto-plus==1.19.6 # via # feast (setup.py) @@ -351,7 +490,7 @@ proto-plus==1.19.6 # google-cloud-bigquery-storage # google-cloud-datastore # google-cloud-firestore -protobuf==3.19.3 +protobuf==3.19.4 # via # feast (setup.py) # google-api-core @@ -363,6 +502,10 @@ protobuf==3.19.3 # mypy-protobuf # proto-plus # tensorflow-metadata +ptyprocess==0.7.0 + # via + # pexpect + # terminado py==1.11.0 # via # pytest @@ -383,7 +526,7 @@ pycodestyle==2.8.0 # via flake8 pycparser==2.21 # via cffi -pycryptodomex==3.13.0 +pycryptodomex==3.14.0 # via snowflake-connector-python pydantic==1.9.0 # via @@ -392,7 +535,11 @@ pydantic==1.9.0 pyflakes==2.4.0 # via flake8 pygments==2.11.2 - # via sphinx + # via + # ipython + # jupyterlab-pygments + # nbconvert + # sphinx pyjwt[crypto]==2.3.0 # via # adal @@ -400,8 +547,9 @@ pyjwt[crypto]==2.3.0 # snowflake-connector-python pyopenssl==21.0.0 # via snowflake-connector-python -pyparsing==3.0.7 +pyparsing==2.4.7 # via + # great-expectations # httplib2 # packaging pyrsistent==0.18.1 @@ -438,6 +586,8 @@ python-dateutil==2.8.2 # adal # botocore # google-cloud-bigquery + # great-expectations + # jupyter-client # moto # pandas python-dotenv==0.19.2 @@ -446,16 +596,23 @@ pytz==2021.3 # via # babel # google-api-core + # great-expectations # moto # pandas # snowflake-connector-python +pytz-deprecation-shim==0.1.0.post0 + # via tzlocal pyyaml==6.0 # via # feast (setup.py) # libcst # pre-commit # uvicorn -redis==4.1.1 +pyzmq==22.3.0 + # via + # jupyter-client + # notebook +redis==4.1.2 # via feast (setup.py) regex==2022.1.18 # via black @@ -471,6 +628,7 @@ requests==2.27.1 # google-api-core # google-cloud-bigquery # google-cloud-storage + # great-expectations # moto # msal # msrest @@ -478,7 +636,7 @@ requests==2.27.1 # responses # snowflake-connector-python # sphinx -requests-oauthlib==1.3.0 +requests-oauthlib==1.3.1 # via # google-auth-oauthlib # msrest @@ -486,13 +644,22 @@ responses==0.17.0 # via moto rsa==4.8 # via google-auth +ruamel.yaml==0.17.17 + # via great-expectations +ruamel.yaml.clib==0.2.6 + # via ruamel.yaml s3transfer==0.5.0 # via boto3 +scipy==1.7.3 + # via great-expectations +send2trash==1.8.0 + # via notebook six==1.16.0 # via # absl-py # azure-core # azure-identity + # bleach # cryptography # google-api-core # google-auth @@ -500,6 +667,7 @@ six==1.16.0 # google-cloud-core # google-resumable-media # grpcio + # isodate # mock # msrestazure # pandavro @@ -539,8 +707,14 @@ tenacity==8.0.1 # via feast (setup.py) tensorflow-metadata==1.6.0 # via feast (setup.py) +termcolor==1.1.0 + # via great-expectations +terminado==0.13.1 + # via notebook testcontainers==3.4.2 # via feast (setup.py) +testpath==0.5.0 + # via nbconvert toml==0.10.2 # via # black @@ -552,40 +726,80 @@ tomli==2.0.0 # coverage # mypy # pep517 +toolz==0.11.2 + # via altair +tornado==6.1 + # via + # ipykernel + # jupyter-client + # notebook + # terminado tqdm==4.62.3 - # via feast (setup.py) -typed-ast==1.5.1 - # via black -types-futures==3.3.7 + # via + # feast (setup.py) + # great-expectations +traitlets==5.1.1 + # via + # ipykernel + # ipython + # ipywidgets + # jupyter-client + # jupyter-core + # matplotlib-inline + # nbclient + # nbconvert + # nbformat + # notebook +typed-ast==1.5.2 + # via + # black + # mypy +types-futures==3.3.8 # via types-protobuf -types-protobuf==3.19.5 +types-protobuf==3.19.7 # via # feast (setup.py) # mypy-protobuf -types-python-dateutil==2.8.8 +types-python-dateutil==2.8.9 # via feast (setup.py) types-pytz==2021.3.4 # via feast (setup.py) -types-pyyaml==6.0.3 +types-pyyaml==6.0.4 # via feast (setup.py) -types-redis==4.1.10 +types-redis==4.1.13 # via feast (setup.py) -types-requests==2.27.7 +types-requests==2.27.8 # via feast (setup.py) -types-setuptools==57.4.7 +types-setuptools==57.4.8 # via feast (setup.py) types-tabulate==0.8.5 # via feast (setup.py) -types-urllib3==1.26.7 +types-urllib3==1.26.8 # via types-requests typing-extensions==4.0.1 # via + # aiohttp + # anyio + # argon2-cffi + # asgiref + # async-timeout + # great-expectations + # h11 + # importlib-metadata + # jsonschema # libcst # mypy # pydantic + # starlette # typing-inspect + # uvicorn + # yarl typing-inspect==0.7.1 # via libcst +tzdata==2021.5 + # via pytz-deprecation-shim +tzlocal==4.1 + # via great-expectations uritemplate==4.1.1 # via google-api-python-client urllib3==1.26.8 @@ -595,7 +809,7 @@ urllib3==1.26.8 # minio # requests # responses -uvicorn[standard]==0.17.0 +uvicorn[standard]==0.17.1 # via feast (setup.py) uvloop==0.16.0 # via uvicorn @@ -603,6 +817,10 @@ virtualenv==20.13.0 # via pre-commit watchgod==0.7 # via uvicorn +wcwidth==0.2.5 + # via prompt-toolkit +webencodings==0.5.1 + # via bleach websocket-client==1.2.3 # via docker websockets==10.1 @@ -611,6 +829,8 @@ werkzeug==2.0.2 # via moto wheel==0.37.1 # via pip-tools +widgetsnbextension==3.5.2 + # via ipywidgets wrapt==1.13.3 # via # deprecated @@ -619,6 +839,11 @@ xmltodict==0.12.0 # via moto yarl==1.7.2 # via aiohttp +zipp==3.7.0 + # via + # importlib-metadata + # importlib-resources + # pep517 # The following packages are considered to be unsafe in a requirements file: # pip diff --git a/sdk/python/setup.py b/sdk/python/setup.py index eda775d66b3..7535987f833 100644 --- a/sdk/python/setup.py +++ b/sdk/python/setup.py @@ -90,12 +90,12 @@ "snowflake-connector-python[pandas]>=2.7.3", ] -DQM_REQUIRED = [ - "great_expectations>=0.14.0" +GE_REQUIRED = [ + "great_expectations>=0.14.0,<0.15.0" ] CI_REQUIRED = ( - [ + [ "cryptography==3.3.2", "flake8", "black==19.10b0", @@ -135,11 +135,11 @@ "types-setuptools", "types-tabulate", ] - + GCP_REQUIRED - + REDIS_REQUIRED - + AWS_REQUIRED - + SNOWFLAKE_REQUIRED - + DQM_REQUIRED + + GCP_REQUIRED + + REDIS_REQUIRED + + AWS_REQUIRED + + SNOWFLAKE_REQUIRED + + GE_REQUIRED ) DEV_REQUIRED = ["mypy-protobuf>=3.1.0", "grpcio-testing==1.*"] + CI_REQUIRED @@ -242,7 +242,7 @@ def run(self): "aws": AWS_REQUIRED, "redis": REDIS_REQUIRED, "snowflake": SNOWFLAKE_REQUIRED, - "dqm": DQM_REQUIRED, + "ge": GE_REQUIRED, }, include_package_data=True, license="Apache", From e37743196234e38ba1b88dc5dce8db7fcb0ddbe2 Mon Sep 17 00:00:00 2001 From: pyalex Date: Tue, 1 Feb 2022 11:20:02 +0800 Subject: [PATCH 11/17] remove RetrievalJobWithValidation Signed-off-by: pyalex --- sdk/python/feast/dqm/utils.py | 73 ------------------- sdk/python/feast/feature_store.py | 10 +-- .../infra/offline_stores/offline_store.py | 54 ++++++++++++-- sdk/python/feast/saved_dataset.py | 4 + .../tests/integration/e2e/test_validation.py | 26 +++---- 5 files changed, 65 insertions(+), 102 deletions(-) delete mode 100644 sdk/python/feast/dqm/utils.py diff --git a/sdk/python/feast/dqm/utils.py b/sdk/python/feast/dqm/utils.py deleted file mode 100644 index 6238b38e4b5..00000000000 --- a/sdk/python/feast/dqm/utils.py +++ /dev/null @@ -1,73 +0,0 @@ -from typing import TYPE_CHECKING, List, Optional - -import pandas as pd -import pyarrow - -from feast.dqm.errors import ValidationFailed -from feast.dqm.profilers.profiler import Profile -from feast.infra.offline_stores.offline_store import RetrievalJob, RetrievalMetadata -from feast.saved_dataset import SavedDatasetStorage, ValidationReference - -if TYPE_CHECKING: - from feast.feature_store import FeatureStore - from feast.on_demand_feature_view import OnDemandFeatureView - - -class RetrievalJobWithValidation(RetrievalJob): - def __init__( - self, - retrieval_job: RetrievalJob, - validation_reference: ValidationReference, - feature_store: "FeatureStore", - ): - self._retrieval_job = retrieval_job - self._validation_reference = validation_reference - self._feature_store = feature_store - - def to_df(self) -> pd.DataFrame: - df = self._retrieval_job.to_df() - - profile = get_reference_profile(self._feature_store, self._validation_reference) - validation_result = profile.validate(df) - if not validation_result.is_success: - raise ValidationFailed(validation_result) - - return df - - def to_arrow(self) -> pyarrow.Table: - table = self._retrieval_job.to_arrow() - - profile = get_reference_profile(self._feature_store, self._validation_reference) - validation_result = profile.validate(table.to_pandas()) - if not validation_result.is_success: - raise ValidationFailed(validation_result) - - return table - - @property - def full_feature_names(self) -> bool: - return self._retrieval_job.full_feature_names - - @property - def on_demand_feature_views(self) -> Optional[List["OnDemandFeatureView"]]: - return self._retrieval_job.on_demand_feature_views - - def _to_df_internal(self) -> pd.DataFrame: - raise NotImplementedError - - def _to_arrow_internal(self) -> pyarrow.Table: - raise NotImplementedError - - def persist(self, storage: SavedDatasetStorage) -> "RetrievalJob": - return self._retrieval_job.persist(storage) - - @property - def metadata(self) -> Optional[RetrievalMetadata]: - return self._retrieval_job.metadata - - -def get_reference_profile( - fs: "FeatureStore", reference: ValidationReference -) -> Profile: - dataset = fs.get_saved_dataset(reference.dataset.name) - return reference.profiler.analyze_dataset(dataset.to_df()) diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index fcd2e4f90f9..6b1dadde5c9 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -41,7 +41,6 @@ from feast.base_feature_view import BaseFeatureView from feast.diff.infra_diff import InfraDiff, diff_infra_protos from feast.diff.registry_diff import RegistryDiff, apply_diff_to_registry, diff_between -from feast.dqm.utils import RetrievalJobWithValidation from feast.entity import Entity from feast.errors import ( EntityNotFoundException, @@ -79,7 +78,7 @@ from feast.repo_config import RepoConfig, load_repo_config from feast.repo_contents import RepoContents from feast.request_feature_view import RequestFeatureView -from feast.saved_dataset import SavedDataset, SavedDatasetStorage, ValidationReference +from feast.saved_dataset import SavedDataset, SavedDatasetStorage from feast.type_map import python_values_to_proto_values from feast.usage import log_exceptions, log_exceptions_and_usage, set_usage_attribute from feast.value_type import ValueType @@ -710,7 +709,6 @@ def get_historical_features( entity_df: Union[pd.DataFrame, str], features: Union[List[str], FeatureService], full_feature_names: bool = False, - validation_reference: Optional[ValidationReference] = None, ) -> RetrievalJob: """Enrich an entity dataframe with historical feature values for either training or batch scoring. @@ -736,7 +734,6 @@ def get_historical_features( full_feature_names: A boolean that provides the option to add the feature view prefixes to the feature names, changing them from the format "feature" to "feature_view__feature" (e.g., "daily_transactions" changes to "customer_fv__daily_transactions"). By default, this value is set to False. - validation_reference: If provided resulting dataset will be validated against this reference profile. Returns: RetrievalJob which can be used to materialize the results. @@ -826,11 +823,6 @@ def get_historical_features( full_feature_names, ) - if validation_reference: - job = RetrievalJobWithValidation( - job, validation_reference, feature_store=self - ) - return job @log_exceptions_and_usage diff --git a/sdk/python/feast/infra/offline_stores/offline_store.py b/sdk/python/feast/infra/offline_stores/offline_store.py index 6a1372b39b3..a52f3e85c90 100644 --- a/sdk/python/feast/infra/offline_stores/offline_store.py +++ b/sdk/python/feast/infra/offline_stores/offline_store.py @@ -11,20 +11,25 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import warnings from abc import ABC, abstractmethod from datetime import datetime -from typing import List, Optional, Union +from typing import TYPE_CHECKING, List, Optional, Union import pandas as pd import pyarrow from feast.data_source import DataSource +from feast.dqm.errors import ValidationFailed from feast.feature_view import FeatureView from feast.on_demand_feature_view import OnDemandFeatureView from feast.registry import Registry from feast.repo_config import RepoConfig from feast.saved_dataset import SavedDatasetStorage +if TYPE_CHECKING: + from feast.saved_dataset import ValidationReference + class RetrievalMetadata: min_event_timestamp: Optional[datetime] @@ -61,8 +66,14 @@ def full_feature_names(self) -> bool: def on_demand_feature_views(self) -> Optional[List[OnDemandFeatureView]]: pass - def to_df(self) -> pd.DataFrame: - """Return dataset as Pandas DataFrame synchronously including on demand transforms""" + def to_df( + self, validation_reference: Optional[ValidationReference] = None + ) -> pd.DataFrame: + """ + Return dataset as Pandas DataFrame synchronously including on demand transforms + Args: + validation_reference: If provided resulting dataset will be validated against this reference profile. + """ features_df = self._to_df_internal() if not self.on_demand_feature_views: return features_df @@ -72,6 +83,19 @@ def to_df(self) -> pd.DataFrame: features_df = features_df.join( odfv.get_transformed_features_df(features_df, self.full_feature_names,) ) + + if validation_reference: + warnings.warn( + "Dataset validation is an experimental feature. " + "This API is unstable and it could and most probably will be changed in the future. " + "We do not guarantee that future changes will maintain backward compatibility.", + RuntimeWarning, + ) + + validation_result = validation_reference.profile.validate(features_df) + if not validation_result.is_success: + raise ValidationFailed(validation_result) + return features_df @abstractmethod @@ -84,8 +108,15 @@ def _to_arrow_internal(self) -> pyarrow.Table: """Return dataset as pyarrow Table synchronously""" pass - def to_arrow(self) -> pyarrow.Table: - """Return dataset as pyarrow Table synchronously""" + def to_arrow( + self, validation_reference: Optional[ValidationReference] = None + ) -> pyarrow.Table: + """ + Return dataset as pyarrow Table synchronously + Args: + validation_reference: If provided resulting dataset will be validated against this reference profile. + + """ if not self.on_demand_feature_views: return self._to_arrow_internal() @@ -94,6 +125,19 @@ def to_arrow(self) -> pyarrow.Table: features_df = features_df.join( odfv.get_transformed_features_df(features_df, self.full_feature_names,) ) + + if validation_reference: + warnings.warn( + "Dataset validation is an experimental feature. " + "This API is unstable and it could and most probably will be changed in the future. " + "We do not guarantee that future changes will maintain backward compatibility.", + RuntimeWarning, + ) + + validation_result = validation_reference.profile.validate(features_df) + if not validation_result.is_success: + raise ValidationFailed(validation_result) + return pyarrow.Table.from_pandas(features_df) @abstractmethod diff --git a/sdk/python/feast/saved_dataset.py b/sdk/python/feast/saved_dataset.py index 64dcffaf803..75b6d2c199f 100644 --- a/sdk/python/feast/saved_dataset.py +++ b/sdk/python/feast/saved_dataset.py @@ -203,3 +203,7 @@ class ValidationReference: def __init__(self, dataset: SavedDataset, profiler: Profiler): self.dataset = dataset self.profiler = profiler + + @property + def profile(self) -> Profile: + return self.profiler.analyze_dataset(self.dataset.to_df()) diff --git a/sdk/python/tests/integration/e2e/test_validation.py b/sdk/python/tests/integration/e2e/test_validation.py index 3996e5fd0a4..2bd1e3cbbc9 100644 --- a/sdk/python/tests/integration/e2e/test_validation.py +++ b/sdk/python/tests/integration/e2e/test_validation.py @@ -81,17 +81,15 @@ def test_historical_retrieval_with_validation(environment, universal_data_source storage=environment.data_source_creator.create_saved_dataset_destination(), ) - job = store.get_historical_features( - entity_df=entity_df, - features=_features, + job = store.get_historical_features(entity_df=entity_df, features=_features,) + + # if validation pass there will be no exceptions on this point + job.to_df( validation_reference=store.get_saved_dataset( "my_training_dataset" - ).as_reference(profiler=configurable_profiler), + ).as_reference(profiler=configurable_profiler) ) - # if validation pass there will be no exceptions on this point - job.to_df() - @pytest.mark.integration @pytest.mark.universal @@ -117,16 +115,14 @@ def test_historical_retrieval_fails_on_validation(environment, universal_data_so storage=environment.data_source_creator.create_saved_dataset_destination(), ) - job = store.get_historical_features( - entity_df=entity_df, - features=_features, - validation_reference=store.get_saved_dataset("my_other_dataset").as_reference( - profiler=profiler_with_unrealistic_expectations - ), - ) + job = store.get_historical_features(entity_df=entity_df, features=_features,) with pytest.raises(ValidationFailed) as exc_info: - job.to_df() + job.to_df( + validation_reference=store.get_saved_dataset( + "my_other_dataset" + ).as_reference(profiler=profiler_with_unrealistic_expectations) + ) failed_expectations = exc_info.value.report.errors assert len(failed_expectations) == 2 From 0f87c7b38537a7f43560dc4add9642cb95b260c1 Mon Sep 17 00:00:00 2001 From: Oleksii Moskalenko Date: Tue, 1 Feb 2022 05:22:27 +0200 Subject: [PATCH 12/17] Update docs/reference/dqm.md Co-authored-by: Danny Chiao Signed-off-by: pyalex --- docs/reference/dqm.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/reference/dqm.md b/docs/reference/dqm.md index bc760928332..a1428132bce 100644 --- a/docs/reference/dqm.md +++ b/docs/reference/dqm.md @@ -16,8 +16,8 @@ Its goal is to address several complex data problems, namely: ### Overview -The validation process consists of the next steps: -1. User prepares reference dataset (currently only [saved dataset](../getting-started/concepts/dataset.md) from historical retrieval is supported). +The validation process consists of the following steps: +1. User prepares reference dataset (currently only [saved datasets](../getting-started/concepts/dataset.md) from historical retrieval are supported). 2. User defines profiler function, which should produce profile by given dataset (currently only profilers based on [Great Expectations](https://docs.greatexpectations.io) are allowed). 3. Validation of tested dataset is performed with reference dataset and profiler provided as parameters. @@ -29,9 +29,9 @@ pip install 'feast[dqm]' ### Dataset profile Currently, Feast supports only [great expectation's](https://greatexpectations.io/) [ExpectationSuite](https://legacy.docs.greatexpectations.io/en/latest/autoapi/great_expectations/core/expectation_suite/index.html#great_expectations.core.expectation_suite.ExpectationSuite) -as dataset's profiler. Hence, the user needs to define a function (profiler) that would receive a dataset and return [ExpectationSuite](https://legacy.docs.greatexpectations.io/en/latest/autoapi/great_expectations/core/expectation_suite/index.html#great_expectations.core.expectation_suite.ExpectationSuite). +as dataset's profiler. Hence, the user needs to define a function (profiler) that would receive a dataset and return an [ExpectationSuite](https://legacy.docs.greatexpectations.io/en/latest/autoapi/great_expectations/core/expectation_suite/index.html#great_expectations.core.expectation_suite.ExpectationSuite). -Either automatic profiler or user selected expectations could be used in profiler function: +Great Expectations supports automatic profiling as well as manually specifying expectations: ```python from great_expectations.dataset import Dataset from great_expectations.core.expectation_suite import ExpectationSuite @@ -59,7 +59,7 @@ def manual_profiler(dataset: Dataset) -> ExpectationSuite: ### Validating Training Dataset -During retrieval of historical features, additional parameter `validation_reference` could be passed. +During retrieval of historical features, `validation_reference` can be passed as a parameter. If this parameter is supplied, `get_historical_features` will return instance `RetrievalJobWithValidation` instead of isntance of `RetrievalJob`. Such job will run validation once dataset is materialized (when `.to_df()` or `.to_arrow()` called). In case if validation successful materialized dataset is returned (no change to previous/regular behavior). Otherwise, `feast.dqm.errors.ValidationFailed` exception would be raised. It will consist of all details for expectations that didn't pass. From 7c8660bfc4c3d734a844feb37ac759dd5e2fb40c Mon Sep 17 00:00:00 2001 From: pyalex Date: Tue, 1 Feb 2022 11:31:05 +0800 Subject: [PATCH 13/17] fix type hint Signed-off-by: pyalex --- sdk/python/feast/infra/offline_stores/offline_store.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/python/feast/infra/offline_stores/offline_store.py b/sdk/python/feast/infra/offline_stores/offline_store.py index a52f3e85c90..56d86ba33df 100644 --- a/sdk/python/feast/infra/offline_stores/offline_store.py +++ b/sdk/python/feast/infra/offline_stores/offline_store.py @@ -67,7 +67,7 @@ def on_demand_feature_views(self) -> Optional[List[OnDemandFeatureView]]: pass def to_df( - self, validation_reference: Optional[ValidationReference] = None + self, validation_reference: Optional["ValidationReference"] = None ) -> pd.DataFrame: """ Return dataset as Pandas DataFrame synchronously including on demand transforms @@ -109,7 +109,7 @@ def _to_arrow_internal(self) -> pyarrow.Table: pass def to_arrow( - self, validation_reference: Optional[ValidationReference] = None + self, validation_reference: Optional["ValidationReference"] = None ) -> pyarrow.Table: """ Return dataset as pyarrow Table synchronously From 80f0814a667f7de0011930bec8c4aca79a20466a Mon Sep 17 00:00:00 2001 From: pyalex Date: Tue, 1 Feb 2022 13:02:17 +0800 Subject: [PATCH 14/17] fix function flow Signed-off-by: pyalex --- docs/reference/dqm.md | 11 +++---- sdk/python/feast/dqm/profilers/ge_profiler.py | 31 +++++++++++++++---- .../infra/offline_stores/offline_store.py | 28 ++++++++++------- sdk/python/tests/__init__.py | 0 sdk/python/tests/data/__init__.py | 0 5 files changed, 46 insertions(+), 24 deletions(-) create mode 100644 sdk/python/tests/__init__.py create mode 100644 sdk/python/tests/data/__init__.py diff --git a/docs/reference/dqm.md b/docs/reference/dqm.md index a1428132bce..61788bf66ac 100644 --- a/docs/reference/dqm.md +++ b/docs/reference/dqm.md @@ -22,13 +22,13 @@ The validation process consists of the following steps: 3. Validation of tested dataset is performed with reference dataset and profiler provided as parameters. ### Preparations -Feast with DQM support can be installed via +Feast with Great Expectations support can be installed via ```shell -pip install 'feast[dqm]' +pip install 'feast[ge]' ``` ### Dataset profile -Currently, Feast supports only [great expectation's](https://greatexpectations.io/) [ExpectationSuite](https://legacy.docs.greatexpectations.io/en/latest/autoapi/great_expectations/core/expectation_suite/index.html#great_expectations.core.expectation_suite.ExpectationSuite) +Currently, Feast supports only [Great Expectation's](https://greatexpectations.io/) [ExpectationSuite](https://legacy.docs.greatexpectations.io/en/latest/autoapi/great_expectations/core/expectation_suite/index.html#great_expectations.core.expectation_suite.ExpectationSuite) as dataset's profiler. Hence, the user needs to define a function (profiler) that would receive a dataset and return an [ExpectationSuite](https://legacy.docs.greatexpectations.io/en/latest/autoapi/great_expectations/core/expectation_suite/index.html#great_expectations.core.expectation_suite.ExpectationSuite). Great Expectations supports automatic profiling as well as manually specifying expectations: @@ -59,9 +59,8 @@ def manual_profiler(dataset: Dataset) -> ExpectationSuite: ### Validating Training Dataset -During retrieval of historical features, `validation_reference` can be passed as a parameter. -If this parameter is supplied, `get_historical_features` will return instance `RetrievalJobWithValidation` instead of isntance of `RetrievalJob`. -Such job will run validation once dataset is materialized (when `.to_df()` or `.to_arrow()` called). In case if validation successful materialized dataset is returned (no change to previous/regular behavior). +During retrieval of historical features, `validation_reference` can be passed as a parameter to methods `.to_df(validation_reference=...)` or `.to_arrow(validation_reference=...)`. +Such job will run validation once dataset is materialized. In case if validation successful materialized dataset is returned. Otherwise, `feast.dqm.errors.ValidationFailed` exception would be raised. It will consist of all details for expectations that didn't pass. ```python diff --git a/sdk/python/feast/dqm/profilers/ge_profiler.py b/sdk/python/feast/dqm/profilers/ge_profiler.py index d90e50eaf3f..28aa6c9fd3b 100644 --- a/sdk/python/feast/dqm/profilers/ge_profiler.py +++ b/sdk/python/feast/dqm/profilers/ge_profiler.py @@ -23,19 +23,22 @@ ) -def _prepare_dataset(dataset): +def _prepare_dataset(dataset: PandasDataset) -> PandasDataset: + dataset_copy = dataset.copy(deep=True) + for column in dataset.columns: if dataset.expect_column_values_to_be_in_type_list( column, type_list=sorted(list(ProfilerTypeMapping.DATETIME_TYPE_NAMES)) ).success: - # GE cannot parse Timestamp or other pandas datetime time - dataset[column] = dataset[column].dt.strftime("%Y-%m-%dT%H:%M:%S") + dataset_copy[column] = dataset[column].dt.strftime("%Y-%m-%dT%H:%M:%S") if dataset[column].dtype == np.float32: # GE converts expectation arguments into native Python float # This could cause error on comparison => so better to convert to double prematurely - dataset[column] = dataset[column].astype(np.float64) + dataset_copy[column] = dataset[column].astype(np.float64) + + return dataset_copy class GEProfile(Profile): @@ -45,9 +48,17 @@ def __init__(self, expectation_suite: ExpectationSuite): self.expectation_suite = expectation_suite def validate(self, df: pd.DataFrame) -> "GEValidationReport": + """ + Validate provided dataframe against GE expectation suite. + 1. Pandas dataframe is converted into PandasDataset + 2. Some fixes applied to avoid crash inside GE (see _prepare_dataset) + 3. Each expectation from ExpectationSuite instance ran against resulting dataset + + Return GEValidationReport, which parses great expectation's schema into list of generic ValidationErrors. + """ dataset = PandasDataset(df) - _prepare_dataset(dataset) + dataset = _prepare_dataset(dataset) results = ge.validate( dataset, expectation_suite=self.expectation_suite, result_format="COMPLETE" @@ -73,9 +84,17 @@ def __init__( self.user_defined_profiler = user_defined_profiler def analyze_dataset(self, df: pd.DataFrame) -> Profile: + """ + Generate GEProfile consisting from ExpectationSuite (set of expectations) + from given pandas dataframe (with user defined profiler). + + Some fixes are also applied to the dataset (see _prepare_dataset function). + + Return GEProfile + """ dataset = PandasDataset(df) - _prepare_dataset(dataset) + dataset = _prepare_dataset(dataset) return GEProfile(expectation_suite=self.user_defined_profiler(dataset)) diff --git a/sdk/python/feast/infra/offline_stores/offline_store.py b/sdk/python/feast/infra/offline_stores/offline_store.py index 56d86ba33df..1e5fe573774 100644 --- a/sdk/python/feast/infra/offline_stores/offline_store.py +++ b/sdk/python/feast/infra/offline_stores/offline_store.py @@ -75,14 +75,15 @@ def to_df( validation_reference: If provided resulting dataset will be validated against this reference profile. """ features_df = self._to_df_internal() - if not self.on_demand_feature_views: - return features_df - # TODO(adchia): Fix requirement to specify dependent feature views in feature_refs - for odfv in self.on_demand_feature_views: - features_df = features_df.join( - odfv.get_transformed_features_df(features_df, self.full_feature_names,) - ) + if self.on_demand_feature_views: + # TODO(adchia): Fix requirement to specify dependent feature views in feature_refs + for odfv in self.on_demand_feature_views: + features_df = features_df.join( + odfv.get_transformed_features_df( + features_df, self.full_feature_names, + ) + ) if validation_reference: warnings.warn( @@ -117,14 +118,17 @@ def to_arrow( validation_reference: If provided resulting dataset will be validated against this reference profile. """ - if not self.on_demand_feature_views: + if not self.on_demand_feature_views and not validation_reference: return self._to_arrow_internal() features_df = self._to_df_internal() - for odfv in self.on_demand_feature_views: - features_df = features_df.join( - odfv.get_transformed_features_df(features_df, self.full_feature_names,) - ) + if self.on_demand_feature_views: + for odfv in self.on_demand_feature_views: + features_df = features_df.join( + odfv.get_transformed_features_df( + features_df, self.full_feature_names, + ) + ) if validation_reference: warnings.warn( diff --git a/sdk/python/tests/__init__.py b/sdk/python/tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/sdk/python/tests/data/__init__.py b/sdk/python/tests/data/__init__.py new file mode 100644 index 00000000000..e69de29bb2d From 34135871c0b121d943d1bf629e6532c5f3b9593b Mon Sep 17 00:00:00 2001 From: pyalex Date: Tue, 1 Feb 2022 13:20:14 +0800 Subject: [PATCH 15/17] more docstrings Signed-off-by: pyalex --- sdk/python/feast/dqm/profilers/ge_profiler.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/sdk/python/feast/dqm/profilers/ge_profiler.py b/sdk/python/feast/dqm/profilers/ge_profiler.py index 28aa6c9fd3b..6c6e22519e2 100644 --- a/sdk/python/feast/dqm/profilers/ge_profiler.py +++ b/sdk/python/feast/dqm/profilers/ge_profiler.py @@ -42,6 +42,11 @@ def _prepare_dataset(dataset: PandasDataset) -> PandasDataset: class GEProfile(Profile): + """ + GEProfile is an implementation of abstract Profile for Great Expectation integration. + It executes validation by applying expectations from ExpectationSuite instance to a given dataset. + """ + expectation_suite: ExpectationSuite def __init__(self, expectation_suite: ExpectationSuite): @@ -78,6 +83,12 @@ def from_proto(cls, proto: GEValidationProfileProto) -> "GEProfile": class GEProfiler(Profiler): + """ + GEProfiler is an implementation of abstract Profiler for Great Expectations integration. + It wraps around user defined profiler that would accept dataset (in a form of pandas dataframe) + and generate GEProfile (with ExpectationSuite inside). + """ + def __init__( self, user_defined_profiler: Callable[[pd.DataFrame], ExpectationSuite] ): From 059444efa8f85f90ca28d488a87d78fefa4d90e8 Mon Sep 17 00:00:00 2001 From: pyalex Date: Tue, 1 Feb 2022 20:24:32 +0800 Subject: [PATCH 16/17] update docs Signed-off-by: pyalex --- docs/reference/dqm.md | 12 ++++++------ sdk/python/feast/dqm/profilers/ge_profiler.py | 6 ++++++ sdk/python/feast/infra/offline_stores/file.py | 4 ++-- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/docs/reference/dqm.md b/docs/reference/dqm.md index 61788bf66ac..5a02413e534 100644 --- a/docs/reference/dqm.md +++ b/docs/reference/dqm.md @@ -29,7 +29,7 @@ pip install 'feast[ge]' ### Dataset profile Currently, Feast supports only [Great Expectation's](https://greatexpectations.io/) [ExpectationSuite](https://legacy.docs.greatexpectations.io/en/latest/autoapi/great_expectations/core/expectation_suite/index.html#great_expectations.core.expectation_suite.ExpectationSuite) -as dataset's profiler. Hence, the user needs to define a function (profiler) that would receive a dataset and return an [ExpectationSuite](https://legacy.docs.greatexpectations.io/en/latest/autoapi/great_expectations/core/expectation_suite/index.html#great_expectations.core.expectation_suite.ExpectationSuite). +as dataset's profile. Hence, the user needs to define a function (profiler) that would receive a dataset and return an [ExpectationSuite](https://legacy.docs.greatexpectations.io/en/latest/autoapi/great_expectations/core/expectation_suite/index.html#great_expectations.core.expectation_suite.ExpectationSuite). Great Expectations supports automatic profiling as well as manually specifying expectations: ```python @@ -48,7 +48,7 @@ def automatic_profiler(dataset: Dataset) -> ExpectationSuite: value_set_threshold='few' ).build_suite() ``` - +However, from our experience capabilities of automatic profiler are quite limited. So we would recommend crafting your own expectations: ```python @ge_profiler def manual_profiler(dataset: Dataset) -> ExpectationSuite: @@ -59,8 +59,8 @@ def manual_profiler(dataset: Dataset) -> ExpectationSuite: ### Validating Training Dataset -During retrieval of historical features, `validation_reference` can be passed as a parameter to methods `.to_df(validation_reference=...)` or `.to_arrow(validation_reference=...)`. -Such job will run validation once dataset is materialized. In case if validation successful materialized dataset is returned. +During retrieval of historical features, `validation_reference` can be passed as a parameter to methods `.to_df(validation_reference=...)` or `.to_arrow(validation_reference=...)` of RetrievalJob. +If parameter is provided Feast will run validation once dataset is materialized. In case if validation successful materialized dataset is returned. Otherwise, `feast.dqm.errors.ValidationFailed` exception would be raised. It will consist of all details for expectations that didn't pass. ```python @@ -68,8 +68,8 @@ from feast import FeatureStore fs = FeatureStore(".") -fs.get_historical_features( - ..., +job = fs.get_historical_features(...) +job.to_df( validation_reference=fs .get_saved_dataset("my_reference_dataset") .as_reference(profiler=manual_profiler) diff --git a/sdk/python/feast/dqm/profilers/ge_profiler.py b/sdk/python/feast/dqm/profilers/ge_profiler.py index 6c6e22519e2..9d1adb5d5e5 100644 --- a/sdk/python/feast/dqm/profilers/ge_profiler.py +++ b/sdk/python/feast/dqm/profilers/ge_profiler.py @@ -81,6 +81,12 @@ def from_proto(cls, proto: GEValidationProfileProto) -> "GEProfile": expectation_suite=ExpectationSuite(**json.loads(proto.expectation_suite)) ) + def __repr__(self): + expectations = json.dumps( + [e.to_json_dict() for e in self.expectation_suite.expectations], indent=2 + ) + return f"" + class GEProfiler(Profiler): """ diff --git a/sdk/python/feast/infra/offline_stores/file.py b/sdk/python/feast/infra/offline_stores/file.py index 6bc8b825389..a49ce643d0b 100644 --- a/sdk/python/feast/infra/offline_stores/file.py +++ b/sdk/python/feast/infra/offline_stores/file.py @@ -83,12 +83,12 @@ def persist(self, storage: SavedDatasetStorage): if path.endswith(".parquet"): pyarrow.parquet.write_table( - self._to_arrow_internal(), where=path, filesystem=filesystem + self.to_arrow(), where=path, filesystem=filesystem ) else: # otherwise assume destination is directory pyarrow.parquet.write_to_dataset( - self._to_arrow_internal(), root_path=path, filesystem=filesystem + self.to_arrow(), root_path=path, filesystem=filesystem ) @property From c8d57c99164c3b4b5f2021df23c77d24ee73fa75 Mon Sep 17 00:00:00 2001 From: pyalex Date: Tue, 1 Feb 2022 20:34:15 +0800 Subject: [PATCH 17/17] improve docs Signed-off-by: pyalex --- sdk/python/feast/dqm/profilers/ge_profiler.py | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/sdk/python/feast/dqm/profilers/ge_profiler.py b/sdk/python/feast/dqm/profilers/ge_profiler.py index 9d1adb5d5e5..f1780754de3 100644 --- a/sdk/python/feast/dqm/profilers/ge_profiler.py +++ b/sdk/python/feast/dqm/profilers/ge_profiler.py @@ -43,7 +43,7 @@ def _prepare_dataset(dataset: PandasDataset) -> PandasDataset: class GEProfile(Profile): """ - GEProfile is an implementation of abstract Profile for Great Expectation integration. + GEProfile is an implementation of abstract Profile for integration with Great Expectations. It executes validation by applying expectations from ExpectationSuite instance to a given dataset. """ @@ -55,9 +55,9 @@ def __init__(self, expectation_suite: ExpectationSuite): def validate(self, df: pd.DataFrame) -> "GEValidationReport": """ Validate provided dataframe against GE expectation suite. - 1. Pandas dataframe is converted into PandasDataset - 2. Some fixes applied to avoid crash inside GE (see _prepare_dataset) - 3. Each expectation from ExpectationSuite instance ran against resulting dataset + 1. Pandas dataframe is converted into PandasDataset (GE type) + 2. Some fixes applied to the data to avoid crashes inside GE (see _prepare_dataset) + 3. Each expectation from ExpectationSuite instance tested against resulting dataset Return GEValidationReport, which parses great expectation's schema into list of generic ValidationErrors. """ @@ -90,9 +90,9 @@ def __repr__(self): class GEProfiler(Profiler): """ - GEProfiler is an implementation of abstract Profiler for Great Expectations integration. - It wraps around user defined profiler that would accept dataset (in a form of pandas dataframe) - and generate GEProfile (with ExpectationSuite inside). + GEProfiler is an implementation of abstract Profiler for integration with Great Expectations. + It wraps around user defined profiler that should accept dataset (in a form of pandas dataframe) + and return ExpectationSuite. """ def __init__( @@ -102,10 +102,10 @@ def __init__( def analyze_dataset(self, df: pd.DataFrame) -> Profile: """ - Generate GEProfile consisting from ExpectationSuite (set of expectations) - from given pandas dataframe (with user defined profiler). + Generate GEProfile with ExpectationSuite (set of expectations) + from a given pandas dataframe by applying user defined profiler. - Some fixes are also applied to the dataset (see _prepare_dataset function). + Some fixes are also applied to the dataset (see _prepare_dataset function) to make it compatible with GE. Return GEProfile """