diff --git a/docs/reference/dqm.md b/docs/reference/dqm.md new file mode 100644 index 00000000000..5a02413e534 --- /dev/null +++ b/docs/reference/dqm.md @@ -0,0 +1,77 @@ +# Data Quality Monitoring + +Data Quality Monitoring (DQM) is a Feast module aimed to help users to validate their data with the user-curated set of rules. +Validation could be applied during: +* Historical retrieval (training dataset generation) +* [planned] Writing features into an online store +* [planned] Reading features from an online store + +Its goal is to address several complex data problems, namely: +* Data consistency - new training datasets can be significantly different from previous datasets. This might require a change in model architecture. +* Issues/bugs in the upstream pipeline - bugs in upstream pipelines can cause invalid values to overwrite existing valid values in an online store. +* Training/serving skew - distribution shift could significantly decrease the performance of the model. + +> To monitor data quality, we check that the characteristics of the tested dataset (aka the tested dataset's profile) are "equivalent" to the characteristics of the reference dataset. +> How exactly profile equivalency should be measured is up to the user. + +### Overview + +The validation process consists of the following steps: +1. User prepares reference dataset (currently only [saved datasets](../getting-started/concepts/dataset.md) from historical retrieval are supported). +2. User defines profiler function, which should produce profile by given dataset (currently only profilers based on [Great Expectations](https://docs.greatexpectations.io) are allowed). +3. Validation of tested dataset is performed with reference dataset and profiler provided as parameters. + +### Preparations +Feast with Great Expectations support can be installed via +```shell +pip install 'feast[ge]' +``` + +### Dataset profile +Currently, Feast supports only [Great Expectation's](https://greatexpectations.io/) [ExpectationSuite](https://legacy.docs.greatexpectations.io/en/latest/autoapi/great_expectations/core/expectation_suite/index.html#great_expectations.core.expectation_suite.ExpectationSuite) +as dataset's profile. Hence, the user needs to define a function (profiler) that would receive a dataset and return an [ExpectationSuite](https://legacy.docs.greatexpectations.io/en/latest/autoapi/great_expectations/core/expectation_suite/index.html#great_expectations.core.expectation_suite.ExpectationSuite). + +Great Expectations supports automatic profiling as well as manually specifying expectations: +```python +from great_expectations.dataset import Dataset +from great_expectations.core.expectation_suite import ExpectationSuite + +from feast.dqm.profilers.ge_profiler import ge_profiler + +@ge_profiler +def automatic_profiler(dataset: Dataset) -> ExpectationSuite: + from great_expectations.profile.user_configurable_profiler import UserConfigurableProfiler + + return UserConfigurableProfiler( + profile_dataset=dataset, + ignored_columns=['conv_rate'], + value_set_threshold='few' + ).build_suite() +``` +However, from our experience capabilities of automatic profiler are quite limited. So we would recommend crafting your own expectations: +```python +@ge_profiler +def manual_profiler(dataset: Dataset) -> ExpectationSuite: + dataset.expect_column_max_to_be_between("column", 1, 2) + return dataset.get_expectation_suite() +``` + + + +### Validating Training Dataset +During retrieval of historical features, `validation_reference` can be passed as a parameter to methods `.to_df(validation_reference=...)` or `.to_arrow(validation_reference=...)` of RetrievalJob. +If parameter is provided Feast will run validation once dataset is materialized. In case if validation successful materialized dataset is returned. +Otherwise, `feast.dqm.errors.ValidationFailed` exception would be raised. It will consist of all details for expectations that didn't pass. + +```python +from feast import FeatureStore + +fs = FeatureStore(".") + +job = fs.get_historical_features(...) +job.to_df( + validation_reference=fs + .get_saved_dataset("my_reference_dataset") + .as_reference(profiler=manual_profiler) +) +``` diff --git a/protos/feast/core/ValidationProfile.proto b/protos/feast/core/ValidationProfile.proto new file mode 100644 index 00000000000..31c4e150a07 --- /dev/null +++ b/protos/feast/core/ValidationProfile.proto @@ -0,0 +1,48 @@ +// +// Copyright 2021 The Feast Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + + +syntax = "proto3"; + +package feast.core; +option java_package = "feast.proto.core"; +option java_outer_classname = "ValidationProfile"; +option go_package = "github.com/feast-dev/feast/sdk/go/protos/feast/core"; + +import "google/protobuf/timestamp.proto"; +import "feast/core/SavedDataset.proto"; + +message GEValidationProfiler { + message UserDefinedProfiler { + // The python-syntax function body (serialized by dill) + bytes body = 1; + } + + UserDefinedProfiler profiler = 1; +} + +message GEValidationProfile { + // JSON-serialized ExpectationSuite object + bytes expectation_suite = 1; +} + +message ValidationReference { + SavedDataset dataset = 1; + + oneof profiler { + GEValidationProfiler ge_profiler = 2; + } +} diff --git a/sdk/python/feast/dqm/__init__.py b/sdk/python/feast/dqm/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/sdk/python/feast/dqm/errors.py b/sdk/python/feast/dqm/errors.py new file mode 100644 index 00000000000..c4179f72b3c --- /dev/null +++ b/sdk/python/feast/dqm/errors.py @@ -0,0 +1,13 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from .profilers.profiler import ValidationReport + + +class ValidationFailed(Exception): + def __init__(self, validation_report: "ValidationReport"): + self.validation_report = validation_report + + @property + def report(self) -> "ValidationReport": + return self.validation_report diff --git a/sdk/python/feast/dqm/profilers/__init__.py b/sdk/python/feast/dqm/profilers/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/sdk/python/feast/dqm/profilers/ge_profiler.py b/sdk/python/feast/dqm/profilers/ge_profiler.py new file mode 100644 index 00000000000..f1780754de3 --- /dev/null +++ b/sdk/python/feast/dqm/profilers/ge_profiler.py @@ -0,0 +1,162 @@ +import json +from typing import Any, Callable, Dict, List + +import dill +import great_expectations as ge +import numpy as np +import pandas as pd +from great_expectations.core import ExpectationSuite +from great_expectations.dataset import PandasDataset +from great_expectations.profile.base import ProfilerTypeMapping + +from feast.dqm.profilers.profiler import ( + Profile, + Profiler, + ValidationError, + ValidationReport, +) +from feast.protos.feast.core.ValidationProfile_pb2 import ( + GEValidationProfile as GEValidationProfileProto, +) +from feast.protos.feast.core.ValidationProfile_pb2 import ( + GEValidationProfiler as GEValidationProfilerProto, +) + + +def _prepare_dataset(dataset: PandasDataset) -> PandasDataset: + dataset_copy = dataset.copy(deep=True) + + for column in dataset.columns: + if dataset.expect_column_values_to_be_in_type_list( + column, type_list=sorted(list(ProfilerTypeMapping.DATETIME_TYPE_NAMES)) + ).success: + # GE cannot parse Timestamp or other pandas datetime time + dataset_copy[column] = dataset[column].dt.strftime("%Y-%m-%dT%H:%M:%S") + + if dataset[column].dtype == np.float32: + # GE converts expectation arguments into native Python float + # This could cause error on comparison => so better to convert to double prematurely + dataset_copy[column] = dataset[column].astype(np.float64) + + return dataset_copy + + +class GEProfile(Profile): + """ + GEProfile is an implementation of abstract Profile for integration with Great Expectations. + It executes validation by applying expectations from ExpectationSuite instance to a given dataset. + """ + + expectation_suite: ExpectationSuite + + def __init__(self, expectation_suite: ExpectationSuite): + self.expectation_suite = expectation_suite + + def validate(self, df: pd.DataFrame) -> "GEValidationReport": + """ + Validate provided dataframe against GE expectation suite. + 1. Pandas dataframe is converted into PandasDataset (GE type) + 2. Some fixes applied to the data to avoid crashes inside GE (see _prepare_dataset) + 3. Each expectation from ExpectationSuite instance tested against resulting dataset + + Return GEValidationReport, which parses great expectation's schema into list of generic ValidationErrors. + """ + dataset = PandasDataset(df) + + dataset = _prepare_dataset(dataset) + + results = ge.validate( + dataset, expectation_suite=self.expectation_suite, result_format="COMPLETE" + ) + return GEValidationReport(results) + + def to_proto(self): + return GEValidationProfileProto( + expectation_suite=json.dumps(self.expectation_suite.to_json_dict()).encode() + ) + + @classmethod + def from_proto(cls, proto: GEValidationProfileProto) -> "GEProfile": + return GEProfile( + expectation_suite=ExpectationSuite(**json.loads(proto.expectation_suite)) + ) + + def __repr__(self): + expectations = json.dumps( + [e.to_json_dict() for e in self.expectation_suite.expectations], indent=2 + ) + return f"" + + +class GEProfiler(Profiler): + """ + GEProfiler is an implementation of abstract Profiler for integration with Great Expectations. + It wraps around user defined profiler that should accept dataset (in a form of pandas dataframe) + and return ExpectationSuite. + """ + + def __init__( + self, user_defined_profiler: Callable[[pd.DataFrame], ExpectationSuite] + ): + self.user_defined_profiler = user_defined_profiler + + def analyze_dataset(self, df: pd.DataFrame) -> Profile: + """ + Generate GEProfile with ExpectationSuite (set of expectations) + from a given pandas dataframe by applying user defined profiler. + + Some fixes are also applied to the dataset (see _prepare_dataset function) to make it compatible with GE. + + Return GEProfile + """ + dataset = PandasDataset(df) + + dataset = _prepare_dataset(dataset) + + return GEProfile(expectation_suite=self.user_defined_profiler(dataset)) + + def to_proto(self): + return GEValidationProfilerProto( + profiler=GEValidationProfilerProto.UserDefinedProfiler( + body=dill.dumps(self.user_defined_profiler, recurse=True) + ) + ) + + @classmethod + def from_proto(cls, proto: GEValidationProfilerProto) -> "GEProfiler": + return GEProfiler(user_defined_profiler=dill.loads(proto.profiler.body)) + + +class GEValidationReport(ValidationReport): + def __init__(self, validation_result: Dict[Any, Any]): + self._validation_result = validation_result + + @property + def is_success(self) -> bool: + return self._validation_result["success"] + + @property + def errors(self) -> List["ValidationError"]: + return [ + ValidationError( + check_name=res.expectation_config.expectation_type, + column_name=res.expectation_config.kwargs["column"], + check_config=res.expectation_config.kwargs, + missing_count=res["result"].get("missing_count"), + missing_percent=res["result"].get("missing_percent"), + ) + for res in self._validation_result["results"] + if not res["success"] + ] + + def __repr__(self): + failed_expectations = [ + res.to_json_dict() + for res in self._validation_result["results"] + if not res["success"] + ] + return json.dumps(failed_expectations, indent=2) + + +def ge_profiler(func): + return GEProfiler(user_defined_profiler=func) diff --git a/sdk/python/feast/dqm/profilers/profiler.py b/sdk/python/feast/dqm/profilers/profiler.py new file mode 100644 index 00000000000..5d2e9d36bc1 --- /dev/null +++ b/sdk/python/feast/dqm/profilers/profiler.py @@ -0,0 +1,88 @@ +import abc +from typing import Any, List, Optional + +import pandas as pd + + +class Profile: + @abc.abstractmethod + def validate(self, dataset: pd.DataFrame) -> "ValidationReport": + """ + Run set of rules / expectations from current profile against given dataset. + + Return ValidationReport + """ + ... + + @abc.abstractmethod + def to_proto(self): + ... + + @classmethod + @abc.abstractmethod + def from_proto(cls, proto) -> "Profile": + ... + + +class Profiler: + @abc.abstractmethod + def analyze_dataset(self, dataset: pd.DataFrame) -> Profile: + """ + Generate Profile object with dataset's characteristics (with rules / expectations) + from given dataset (as pandas dataframe). + """ + ... + + @abc.abstractmethod + def to_proto(self): + ... + + @classmethod + @abc.abstractmethod + def from_proto(cls, proto) -> "Profiler": + ... + + +class ValidationReport: + @property + @abc.abstractmethod + def is_success(self) -> bool: + """ + Return whether validation was successful + """ + ... + + @property + @abc.abstractmethod + def errors(self) -> List["ValidationError"]: + """ + Return list of ValidationErrors if validation failed (is_success = false) + """ + ... + + +class ValidationError: + check_name: str + column_name: str + + check_config: Optional[Any] + + missing_count: Optional[int] + missing_percent: Optional[float] + + def __init__( + self, + check_name: str, + column_name: str, + check_config: Optional[Any] = None, + missing_count: Optional[int] = None, + missing_percent: Optional[float] = None, + ): + self.check_name = check_name + self.column_name = column_name + self.check_config = check_config + self.missing_count = missing_count + self.missing_percent = missing_percent + + def __repr__(self): + return f"" diff --git a/sdk/python/feast/infra/offline_stores/file.py b/sdk/python/feast/infra/offline_stores/file.py index 6bc8b825389..a49ce643d0b 100644 --- a/sdk/python/feast/infra/offline_stores/file.py +++ b/sdk/python/feast/infra/offline_stores/file.py @@ -83,12 +83,12 @@ def persist(self, storage: SavedDatasetStorage): if path.endswith(".parquet"): pyarrow.parquet.write_table( - self._to_arrow_internal(), where=path, filesystem=filesystem + self.to_arrow(), where=path, filesystem=filesystem ) else: # otherwise assume destination is directory pyarrow.parquet.write_to_dataset( - self._to_arrow_internal(), root_path=path, filesystem=filesystem + self.to_arrow(), root_path=path, filesystem=filesystem ) @property diff --git a/sdk/python/feast/infra/offline_stores/offline_store.py b/sdk/python/feast/infra/offline_stores/offline_store.py index 6a1372b39b3..1e5fe573774 100644 --- a/sdk/python/feast/infra/offline_stores/offline_store.py +++ b/sdk/python/feast/infra/offline_stores/offline_store.py @@ -11,20 +11,25 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import warnings from abc import ABC, abstractmethod from datetime import datetime -from typing import List, Optional, Union +from typing import TYPE_CHECKING, List, Optional, Union import pandas as pd import pyarrow from feast.data_source import DataSource +from feast.dqm.errors import ValidationFailed from feast.feature_view import FeatureView from feast.on_demand_feature_view import OnDemandFeatureView from feast.registry import Registry from feast.repo_config import RepoConfig from feast.saved_dataset import SavedDatasetStorage +if TYPE_CHECKING: + from feast.saved_dataset import ValidationReference + class RetrievalMetadata: min_event_timestamp: Optional[datetime] @@ -61,17 +66,37 @@ def full_feature_names(self) -> bool: def on_demand_feature_views(self) -> Optional[List[OnDemandFeatureView]]: pass - def to_df(self) -> pd.DataFrame: - """Return dataset as Pandas DataFrame synchronously including on demand transforms""" + def to_df( + self, validation_reference: Optional["ValidationReference"] = None + ) -> pd.DataFrame: + """ + Return dataset as Pandas DataFrame synchronously including on demand transforms + Args: + validation_reference: If provided resulting dataset will be validated against this reference profile. + """ features_df = self._to_df_internal() - if not self.on_demand_feature_views: - return features_df - # TODO(adchia): Fix requirement to specify dependent feature views in feature_refs - for odfv in self.on_demand_feature_views: - features_df = features_df.join( - odfv.get_transformed_features_df(features_df, self.full_feature_names,) + if self.on_demand_feature_views: + # TODO(adchia): Fix requirement to specify dependent feature views in feature_refs + for odfv in self.on_demand_feature_views: + features_df = features_df.join( + odfv.get_transformed_features_df( + features_df, self.full_feature_names, + ) + ) + + if validation_reference: + warnings.warn( + "Dataset validation is an experimental feature. " + "This API is unstable and it could and most probably will be changed in the future. " + "We do not guarantee that future changes will maintain backward compatibility.", + RuntimeWarning, ) + + validation_result = validation_reference.profile.validate(features_df) + if not validation_result.is_success: + raise ValidationFailed(validation_result) + return features_df @abstractmethod @@ -84,16 +109,39 @@ def _to_arrow_internal(self) -> pyarrow.Table: """Return dataset as pyarrow Table synchronously""" pass - def to_arrow(self) -> pyarrow.Table: - """Return dataset as pyarrow Table synchronously""" - if not self.on_demand_feature_views: + def to_arrow( + self, validation_reference: Optional["ValidationReference"] = None + ) -> pyarrow.Table: + """ + Return dataset as pyarrow Table synchronously + Args: + validation_reference: If provided resulting dataset will be validated against this reference profile. + + """ + if not self.on_demand_feature_views and not validation_reference: return self._to_arrow_internal() features_df = self._to_df_internal() - for odfv in self.on_demand_feature_views: - features_df = features_df.join( - odfv.get_transformed_features_df(features_df, self.full_feature_names,) + if self.on_demand_feature_views: + for odfv in self.on_demand_feature_views: + features_df = features_df.join( + odfv.get_transformed_features_df( + features_df, self.full_feature_names, + ) + ) + + if validation_reference: + warnings.warn( + "Dataset validation is an experimental feature. " + "This API is unstable and it could and most probably will be changed in the future. " + "We do not guarantee that future changes will maintain backward compatibility.", + RuntimeWarning, ) + + validation_result = validation_reference.profile.validate(features_df) + if not validation_result.is_success: + raise ValidationFailed(validation_result) + return pyarrow.Table.from_pandas(features_df) @abstractmethod diff --git a/sdk/python/feast/saved_dataset.py b/sdk/python/feast/saved_dataset.py index 39708685795..75b6d2c199f 100644 --- a/sdk/python/feast/saved_dataset.py +++ b/sdk/python/feast/saved_dataset.py @@ -7,6 +7,7 @@ from google.protobuf.json_format import MessageToJson from feast.data_source import DataSource +from feast.dqm.profilers.profiler import Profile, Profiler from feast.protos.feast.core.SavedDataset_pb2 import SavedDataset as SavedDatasetProto from feast.protos.feast.core.SavedDataset_pb2 import SavedDatasetMeta, SavedDatasetSpec from feast.protos.feast.core.SavedDataset_pb2 import ( @@ -60,6 +61,8 @@ class SavedDataset: min_event_timestamp: Optional[datetime] = None max_event_timestamp: Optional[datetime] = None + _retrieval_job: Optional["RetrievalJob"] = None + def __init__( self, name: str, @@ -76,6 +79,8 @@ def __init__( self.full_feature_names = full_feature_names self.tags = tags or {} + self._retrieval_job = None + def __repr__(self): items = (f"{k} = {v}" for k, v in self.__dict__.items()) return f"<{self.__class__.__name__}({', '.join(items)})>" @@ -183,3 +188,22 @@ def to_arrow(self) -> pyarrow.Table: ) return self._retrieval_job.to_arrow() + + def as_reference(self, profiler: "Profiler") -> "ValidationReference": + return ValidationReference(profiler=profiler, dataset=self) + + def get_profile(self, profiler: Profiler) -> Profile: + return profiler.analyze_dataset(self.to_df()) + + +class ValidationReference: + dataset: SavedDataset + profiler: Profiler + + def __init__(self, dataset: SavedDataset, profiler: Profiler): + self.dataset = dataset + self.profiler = profiler + + @property + def profile(self) -> Profile: + return self.profiler.analyze_dataset(self.dataset.to_df()) diff --git a/sdk/python/requirements/py3.7-ci-requirements.txt b/sdk/python/requirements/py3.7-ci-requirements.txt index 293b44e0531..d5f654e515c 100644 --- a/sdk/python/requirements/py3.7-ci-requirements.txt +++ b/sdk/python/requirements/py3.7-ci-requirements.txt @@ -20,11 +20,21 @@ aiosignal==1.2.0 # via aiohttp alabaster==0.7.12 # via sphinx +altair==4.2.0 + # via great-expectations anyio==3.5.0 # via starlette appdirs==1.4.4 # via black -asgiref==3.4.1 +appnope==0.1.2 + # via + # ipykernel + # ipython +argon2-cffi==21.3.0 + # via notebook +argon2-cffi-bindings==21.2.0 + # via argon2-cffi +asgiref==3.5.0 # via uvicorn asn1crypto==1.4.0 # via @@ -57,13 +67,21 @@ azure-storage-blob==12.9.0 # via adlfs babel==2.9.1 # via sphinx +backcall==0.2.0 + # via ipython +backports.zoneinfo==0.2.1 + # via + # pytz-deprecation-shim + # tzlocal black==19.10b0 # via feast (setup.py) -boto3==1.20.40 +bleach==4.1.0 + # via nbconvert +boto3==1.20.46 # via # feast (setup.py) # moto -botocore==1.23.40 +botocore==1.23.46 # via # boto3 # moto @@ -80,12 +98,13 @@ certifi==2021.10.8 # snowflake-connector-python cffi==1.15.0 # via + # argon2-cffi-bindings # azure-datalake-store # cryptography # snowflake-connector-python cfgv==3.3.1 # via pre-commit -charset-normalizer==2.0.10 +charset-normalizer==2.0.11 # via # aiohttp # requests @@ -94,11 +113,12 @@ click==8.0.3 # via # black # feast (setup.py) + # great-expectations # pip-tools # uvicorn colorama==0.4.4 # via feast (setup.py) -coverage[toml]==6.2 +coverage[toml]==6.3 # via pytest-cov cryptography==3.3.2 # via @@ -108,10 +128,17 @@ cryptography==3.3.2 # feast (setup.py) # moto # msal + # pyjwt # pyopenssl # snowflake-connector-python +debugpy==1.5.1 + # via ipykernel decorator==5.1.1 - # via gcsfs + # via + # gcsfs + # ipython +defusedxml==0.7.1 + # via nbconvert deprecated==1.2.13 # via redis deprecation==2.1.0 @@ -128,9 +155,14 @@ docutils==0.17.1 # via # sphinx # sphinx-rtd-theme +entrypoints==0.3 + # via + # altair + # jupyter-client + # nbconvert execnet==1.9.0 # via pytest-xdist -fastapi==0.72.0 +fastapi==0.73.0 # via feast (setup.py) fastavro==1.4.9 # via @@ -208,6 +240,8 @@ googleapis-common-protos==1.52.0 # feast (setup.py) # google-api-core # tensorflow-metadata +great-expectations==0.14.4 + # via feast (setup.py) grpcio==1.43.0 # via # feast (setup.py) @@ -232,7 +266,7 @@ httplib2==0.20.2 # google-auth-httplib2 httptools==0.3.0 # via uvicorn -identify==2.4.4 +identify==2.4.7 # via pre-commit idna==3.3 # via @@ -246,6 +280,7 @@ importlib-metadata==4.2.0 # via # click # flake8 + # great-expectations # jsonschema # moto # pep517 @@ -258,22 +293,66 @@ importlib-resources==5.4.0 # via jsonschema iniconfig==1.1.1 # via pytest +ipykernel==6.7.0 + # via + # ipywidgets + # notebook +ipython==7.31.1 + # via + # ipykernel + # ipywidgets +ipython-genutils==0.2.0 + # via + # ipywidgets + # nbformat + # notebook +ipywidgets==7.6.5 + # via great-expectations isodate==0.6.1 # via msrest isort==5.10.1 # via feast (setup.py) +jedi==0.18.1 + # via ipython jinja2==3.0.3 # via + # altair # feast (setup.py) + # great-expectations # moto + # nbconvert + # notebook # sphinx jmespath==0.10.0 # via # boto3 # botocore +jsonpatch==1.32 + # via great-expectations +jsonpointer==2.2 + # via jsonpatch jsonschema==4.4.0 - # via feast (setup.py) -libcst==0.4.0 + # via + # altair + # feast (setup.py) + # great-expectations + # nbformat +jupyter-client==7.1.2 + # via + # ipykernel + # nbclient + # notebook +jupyter-core==4.9.1 + # via + # jupyter-client + # nbconvert + # nbformat + # notebook +jupyterlab-pygments==0.1.2 + # via nbconvert +jupyterlab-widgets==1.0.2 + # via ipywidgets +libcst==0.4.1 # via # google-cloud-bigquery-storage # google-cloud-datastore @@ -281,15 +360,23 @@ markupsafe==2.0.1 # via # jinja2 # moto +matplotlib-inline==0.1.3 + # via + # ipykernel + # ipython mccabe==0.6.1 # via flake8 minio==7.1.0 # via feast (setup.py) +mistune==0.8.4 + # via + # great-expectations + # nbconvert mmh3==3.0.0 # via feast (setup.py) mock==2.0.0 # via feast (setup.py) -moto==3.0.0 +moto==3.0.2 # via feast (setup.py) msal==1.16.0 # via @@ -305,7 +392,7 @@ msrest==0.6.21 # msrestazure msrestazure==0.6.4 # via adlfs -multidict==5.2.0 +multidict==6.0.2 # via # aiohttp # yarl @@ -317,19 +404,41 @@ mypy-extensions==0.4.3 # typing-inspect mypy-protobuf==3.1.0 # via feast (setup.py) +nbclient==0.5.10 + # via nbconvert +nbconvert==6.4.1 + # via notebook +nbformat==5.1.3 + # via + # ipywidgets + # nbclient + # nbconvert + # notebook +nest-asyncio==1.5.4 + # via + # ipykernel + # jupyter-client + # nbclient + # notebook nodeenv==1.6.0 # via pre-commit +notebook==6.4.8 + # via widgetsnbextension numpy==1.21.5 # via + # altair + # great-expectations # pandas # pandavro # pyarrow -oauthlib==3.1.1 + # scipy +oauthlib==3.2.0 # via requests-oauthlib oscrypto==1.2.1 # via snowflake-connector-python packaging==21.3 # via + # bleach # deprecation # google-api-core # google-cloud-bigquery @@ -339,17 +448,27 @@ packaging==21.3 # sphinx pandas==1.3.5 # via + # altair # feast (setup.py) + # great-expectations # pandavro # snowflake-connector-python pandavro==1.5.2 # via feast (setup.py) +pandocfilters==1.5.0 + # via nbconvert +parso==0.8.3 + # via jedi pathspec==0.9.0 # via black pbr==5.8.0 # via mock pep517==0.12.0 # via pip-tools +pexpect==4.8.0 + # via ipython +pickleshare==0.7.5 + # via ipython pip-tools==6.4.0 # via feast (setup.py) platformdirs==2.4.1 @@ -360,6 +479,10 @@ portalocker==2.3.2 # via msal-extensions pre-commit==2.17.0 # via feast (setup.py) +prometheus-client==0.13.1 + # via notebook +prompt-toolkit==3.0.26 + # via ipython proto-plus==1.19.6 # via # feast (setup.py) @@ -367,7 +490,7 @@ proto-plus==1.19.6 # google-cloud-bigquery-storage # google-cloud-datastore # google-cloud-firestore -protobuf==3.19.3 +protobuf==3.19.4 # via # feast (setup.py) # google-api-core @@ -379,6 +502,10 @@ protobuf==3.19.3 # mypy-protobuf # proto-plus # tensorflow-metadata +ptyprocess==0.7.0 + # via + # pexpect + # terminado py==1.11.0 # via # pytest @@ -399,7 +526,7 @@ pycodestyle==2.8.0 # via flake8 pycparser==2.21 # via cffi -pycryptodomex==3.13.0 +pycryptodomex==3.14.0 # via snowflake-connector-python pydantic==1.9.0 # via @@ -408,7 +535,11 @@ pydantic==1.9.0 pyflakes==2.4.0 # via flake8 pygments==2.11.2 - # via sphinx + # via + # ipython + # jupyterlab-pygments + # nbconvert + # sphinx pyjwt[crypto]==2.3.0 # via # adal @@ -416,8 +547,9 @@ pyjwt[crypto]==2.3.0 # snowflake-connector-python pyopenssl==21.0.0 # via snowflake-connector-python -pyparsing==3.0.7 +pyparsing==2.4.7 # via + # great-expectations # httplib2 # packaging pyrsistent==0.18.1 @@ -454,6 +586,8 @@ python-dateutil==2.8.2 # adal # botocore # google-cloud-bigquery + # great-expectations + # jupyter-client # moto # pandas python-dotenv==0.19.2 @@ -462,16 +596,23 @@ pytz==2021.3 # via # babel # google-api-core + # great-expectations # moto # pandas # snowflake-connector-python +pytz-deprecation-shim==0.1.0.post0 + # via tzlocal pyyaml==6.0 # via # feast (setup.py) # libcst # pre-commit # uvicorn -redis==4.1.1 +pyzmq==22.3.0 + # via + # jupyter-client + # notebook +redis==4.1.2 # via feast (setup.py) regex==2022.1.18 # via black @@ -487,6 +628,7 @@ requests==2.27.1 # google-api-core # google-cloud-bigquery # google-cloud-storage + # great-expectations # moto # msal # msrest @@ -494,7 +636,7 @@ requests==2.27.1 # responses # snowflake-connector-python # sphinx -requests-oauthlib==1.3.0 +requests-oauthlib==1.3.1 # via # google-auth-oauthlib # msrest @@ -502,13 +644,22 @@ responses==0.17.0 # via moto rsa==4.8 # via google-auth +ruamel.yaml==0.17.17 + # via great-expectations +ruamel.yaml.clib==0.2.6 + # via ruamel.yaml s3transfer==0.5.0 # via boto3 +scipy==1.7.3 + # via great-expectations +send2trash==1.8.0 + # via notebook six==1.16.0 # via # absl-py # azure-core # azure-identity + # bleach # cryptography # google-api-core # google-auth @@ -516,6 +667,7 @@ six==1.16.0 # google-cloud-core # google-resumable-media # grpcio + # isodate # mock # msrestazure # pandavro @@ -555,8 +707,14 @@ tenacity==8.0.1 # via feast (setup.py) tensorflow-metadata==1.6.0 # via feast (setup.py) +termcolor==1.1.0 + # via great-expectations +terminado==0.13.1 + # via notebook testcontainers==3.4.2 # via feast (setup.py) +testpath==0.5.0 + # via nbconvert toml==0.10.2 # via # black @@ -568,40 +726,64 @@ tomli==2.0.0 # coverage # mypy # pep517 +toolz==0.11.2 + # via altair +tornado==6.1 + # via + # ipykernel + # jupyter-client + # notebook + # terminado tqdm==4.62.3 - # via feast (setup.py) -typed-ast==1.5.1 + # via + # feast (setup.py) + # great-expectations +traitlets==5.1.1 + # via + # ipykernel + # ipython + # ipywidgets + # jupyter-client + # jupyter-core + # matplotlib-inline + # nbclient + # nbconvert + # nbformat + # notebook +typed-ast==1.5.2 # via # black # mypy -types-futures==3.3.7 +types-futures==3.3.8 # via types-protobuf -types-protobuf==3.19.5 +types-protobuf==3.19.7 # via # feast (setup.py) # mypy-protobuf -types-python-dateutil==2.8.8 +types-python-dateutil==2.8.9 # via feast (setup.py) types-pytz==2021.3.4 # via feast (setup.py) -types-pyyaml==6.0.3 +types-pyyaml==6.0.4 # via feast (setup.py) -types-redis==4.1.10 +types-redis==4.1.13 # via feast (setup.py) -types-requests==2.27.7 +types-requests==2.27.8 # via feast (setup.py) -types-setuptools==57.4.7 +types-setuptools==57.4.8 # via feast (setup.py) types-tabulate==0.8.5 # via feast (setup.py) -types-urllib3==1.26.7 +types-urllib3==1.26.8 # via types-requests typing-extensions==4.0.1 # via # aiohttp # anyio + # argon2-cffi # asgiref # async-timeout + # great-expectations # h11 # importlib-metadata # jsonschema @@ -614,6 +796,10 @@ typing-extensions==4.0.1 # yarl typing-inspect==0.7.1 # via libcst +tzdata==2021.5 + # via pytz-deprecation-shim +tzlocal==4.1 + # via great-expectations uritemplate==4.1.1 # via google-api-python-client urllib3==1.26.8 @@ -623,7 +809,7 @@ urllib3==1.26.8 # minio # requests # responses -uvicorn[standard]==0.17.0 +uvicorn[standard]==0.17.1 # via feast (setup.py) uvloop==0.16.0 # via uvicorn @@ -631,6 +817,10 @@ virtualenv==20.13.0 # via pre-commit watchgod==0.7 # via uvicorn +wcwidth==0.2.5 + # via prompt-toolkit +webencodings==0.5.1 + # via bleach websocket-client==1.2.3 # via docker websockets==10.1 @@ -639,6 +829,8 @@ werkzeug==2.0.2 # via moto wheel==0.37.1 # via pip-tools +widgetsnbextension==3.5.2 + # via ipywidgets wrapt==1.13.3 # via # deprecated diff --git a/sdk/python/requirements/py3.8-ci-requirements.txt b/sdk/python/requirements/py3.8-ci-requirements.txt index 3cdc118144f..7a94294c956 100644 --- a/sdk/python/requirements/py3.8-ci-requirements.txt +++ b/sdk/python/requirements/py3.8-ci-requirements.txt @@ -20,11 +20,21 @@ aiosignal==1.2.0 # via aiohttp alabaster==0.7.12 # via sphinx +altair==4.2.0 + # via great-expectations anyio==3.5.0 # via starlette appdirs==1.4.4 # via black -asgiref==3.4.1 +appnope==0.1.2 + # via + # ipykernel + # ipython +argon2-cffi==21.3.0 + # via notebook +argon2-cffi-bindings==21.2.0 + # via argon2-cffi +asgiref==3.5.0 # via uvicorn asn1crypto==1.4.0 # via @@ -34,6 +44,8 @@ assertpy==1.1 # via feast (setup.py) async-timeout==4.0.2 # via aiohttp +asynctest==0.13.0 + # via aiohttp attrs==21.4.0 # via # aiohttp @@ -55,13 +67,21 @@ azure-storage-blob==12.9.0 # via adlfs babel==2.9.1 # via sphinx +backcall==0.2.0 + # via ipython +backports.zoneinfo==0.2.1 + # via + # pytz-deprecation-shim + # tzlocal black==19.10b0 # via feast (setup.py) -boto3==1.20.40 +bleach==4.1.0 + # via nbconvert +boto3==1.20.46 # via # feast (setup.py) # moto -botocore==1.23.40 +botocore==1.23.46 # via # boto3 # moto @@ -78,12 +98,13 @@ certifi==2021.10.8 # snowflake-connector-python cffi==1.15.0 # via + # argon2-cffi-bindings # azure-datalake-store # cryptography # snowflake-connector-python cfgv==3.3.1 # via pre-commit -charset-normalizer==2.0.10 +charset-normalizer==2.0.11 # via # aiohttp # requests @@ -92,11 +113,12 @@ click==8.0.3 # via # black # feast (setup.py) + # great-expectations # pip-tools # uvicorn colorama==0.4.4 # via feast (setup.py) -coverage[toml]==6.2 +coverage[toml]==6.3 # via pytest-cov cryptography==3.3.2 # via @@ -106,10 +128,17 @@ cryptography==3.3.2 # feast (setup.py) # moto # msal + # pyjwt # pyopenssl # snowflake-connector-python +debugpy==1.5.1 + # via ipykernel decorator==5.1.1 - # via gcsfs + # via + # gcsfs + # ipython +defusedxml==0.7.1 + # via nbconvert deprecated==1.2.13 # via redis deprecation==2.1.0 @@ -126,9 +155,14 @@ docutils==0.17.1 # via # sphinx # sphinx-rtd-theme +entrypoints==0.3 + # via + # altair + # jupyter-client + # nbconvert execnet==1.9.0 # via pytest-xdist -fastapi==0.72.0 +fastapi==0.73.0 # via feast (setup.py) fastavro==1.4.9 # via @@ -206,6 +240,8 @@ googleapis-common-protos==1.52.0 # feast (setup.py) # google-api-core # tensorflow-metadata +great-expectations==0.14.4 + # via feast (setup.py) grpcio==1.43.0 # via # feast (setup.py) @@ -230,7 +266,7 @@ httplib2==0.20.2 # google-auth-httplib2 httptools==0.3.0 # via uvicorn -identify==2.4.4 +identify==2.4.7 # via pre-commit idna==3.3 # via @@ -240,26 +276,83 @@ idna==3.3 # yarl imagesize==1.3.0 # via sphinx +importlib-metadata==4.2.0 + # via + # click + # flake8 + # great-expectations + # jsonschema + # moto + # pep517 + # pluggy + # pre-commit + # pytest + # redis + # virtualenv importlib-resources==5.4.0 # via jsonschema iniconfig==1.1.1 # via pytest +ipykernel==6.7.0 + # via + # ipywidgets + # notebook +ipython==7.31.1 + # via + # ipykernel + # ipywidgets +ipython-genutils==0.2.0 + # via + # ipywidgets + # nbformat + # notebook +ipywidgets==7.6.5 + # via great-expectations isodate==0.6.1 # via msrest isort==5.10.1 # via feast (setup.py) +jedi==0.18.1 + # via ipython jinja2==3.0.3 # via + # altair # feast (setup.py) + # great-expectations # moto + # nbconvert + # notebook # sphinx jmespath==0.10.0 # via # boto3 # botocore +jsonpatch==1.32 + # via great-expectations +jsonpointer==2.2 + # via jsonpatch jsonschema==4.4.0 - # via feast (setup.py) -libcst==0.4.0 + # via + # altair + # feast (setup.py) + # great-expectations + # nbformat +jupyter-client==7.1.2 + # via + # ipykernel + # nbclient + # notebook +jupyter-core==4.9.1 + # via + # jupyter-client + # nbconvert + # nbformat + # notebook +jupyterlab-pygments==0.1.2 + # via nbconvert +jupyterlab-widgets==1.0.2 + # via ipywidgets +libcst==0.4.1 # via # google-cloud-bigquery-storage # google-cloud-datastore @@ -267,15 +360,23 @@ markupsafe==2.0.1 # via # jinja2 # moto +matplotlib-inline==0.1.3 + # via + # ipykernel + # ipython mccabe==0.6.1 # via flake8 minio==7.1.0 # via feast (setup.py) +mistune==0.8.4 + # via + # great-expectations + # nbconvert mmh3==3.0.0 # via feast (setup.py) mock==2.0.0 # via feast (setup.py) -moto==3.0.0 +moto==3.0.2 # via feast (setup.py) msal==1.16.0 # via @@ -291,7 +392,7 @@ msrest==0.6.21 # msrestazure msrestazure==0.6.4 # via adlfs -multidict==5.2.0 +multidict==6.0.2 # via # aiohttp # yarl @@ -303,19 +404,41 @@ mypy-extensions==0.4.3 # typing-inspect mypy-protobuf==3.1.0 # via feast (setup.py) +nbclient==0.5.10 + # via nbconvert +nbconvert==6.4.1 + # via notebook +nbformat==5.1.3 + # via + # ipywidgets + # nbclient + # nbconvert + # notebook +nest-asyncio==1.5.4 + # via + # ipykernel + # jupyter-client + # nbclient + # notebook nodeenv==1.6.0 # via pre-commit -numpy==1.22.1 +notebook==6.4.8 + # via widgetsnbextension +numpy==1.21.5 # via + # altair + # great-expectations # pandas # pandavro # pyarrow -oauthlib==3.1.1 + # scipy +oauthlib==3.2.0 # via requests-oauthlib oscrypto==1.2.1 # via snowflake-connector-python packaging==21.3 # via + # bleach # deprecation # google-api-core # google-cloud-bigquery @@ -325,17 +448,27 @@ packaging==21.3 # sphinx pandas==1.3.5 # via + # altair # feast (setup.py) + # great-expectations # pandavro # snowflake-connector-python pandavro==1.5.2 # via feast (setup.py) +pandocfilters==1.5.0 + # via nbconvert +parso==0.8.3 + # via jedi pathspec==0.9.0 # via black pbr==5.8.0 # via mock pep517==0.12.0 # via pip-tools +pexpect==4.8.0 + # via ipython +pickleshare==0.7.5 + # via ipython pip-tools==6.4.0 # via feast (setup.py) platformdirs==2.4.1 @@ -346,6 +479,10 @@ portalocker==2.3.2 # via msal-extensions pre-commit==2.17.0 # via feast (setup.py) +prometheus-client==0.13.1 + # via notebook +prompt-toolkit==3.0.26 + # via ipython proto-plus==1.19.6 # via # feast (setup.py) @@ -353,7 +490,7 @@ proto-plus==1.19.6 # google-cloud-bigquery-storage # google-cloud-datastore # google-cloud-firestore -protobuf==3.19.3 +protobuf==3.19.4 # via # feast (setup.py) # google-api-core @@ -365,6 +502,10 @@ protobuf==3.19.3 # mypy-protobuf # proto-plus # tensorflow-metadata +ptyprocess==0.7.0 + # via + # pexpect + # terminado py==1.11.0 # via # pytest @@ -385,7 +526,7 @@ pycodestyle==2.8.0 # via flake8 pycparser==2.21 # via cffi -pycryptodomex==3.13.0 +pycryptodomex==3.14.0 # via snowflake-connector-python pydantic==1.9.0 # via @@ -394,7 +535,11 @@ pydantic==1.9.0 pyflakes==2.4.0 # via flake8 pygments==2.11.2 - # via sphinx + # via + # ipython + # jupyterlab-pygments + # nbconvert + # sphinx pyjwt[crypto]==2.3.0 # via # adal @@ -402,8 +547,9 @@ pyjwt[crypto]==2.3.0 # snowflake-connector-python pyopenssl==21.0.0 # via snowflake-connector-python -pyparsing==3.0.7 +pyparsing==2.4.7 # via + # great-expectations # httplib2 # packaging pyrsistent==0.18.1 @@ -440,6 +586,8 @@ python-dateutil==2.8.2 # adal # botocore # google-cloud-bigquery + # great-expectations + # jupyter-client # moto # pandas python-dotenv==0.19.2 @@ -448,16 +596,23 @@ pytz==2021.3 # via # babel # google-api-core + # great-expectations # moto # pandas # snowflake-connector-python +pytz-deprecation-shim==0.1.0.post0 + # via tzlocal pyyaml==6.0 # via # feast (setup.py) # libcst # pre-commit # uvicorn -redis==4.1.1 +pyzmq==22.3.0 + # via + # jupyter-client + # notebook +redis==4.1.2 # via feast (setup.py) regex==2022.1.18 # via black @@ -473,6 +628,7 @@ requests==2.27.1 # google-api-core # google-cloud-bigquery # google-cloud-storage + # great-expectations # moto # msal # msrest @@ -480,7 +636,7 @@ requests==2.27.1 # responses # snowflake-connector-python # sphinx -requests-oauthlib==1.3.0 +requests-oauthlib==1.3.1 # via # google-auth-oauthlib # msrest @@ -488,13 +644,22 @@ responses==0.17.0 # via moto rsa==4.8 # via google-auth +ruamel.yaml==0.17.17 + # via great-expectations +ruamel.yaml.clib==0.2.6 + # via ruamel.yaml s3transfer==0.5.0 # via boto3 +scipy==1.7.3 + # via great-expectations +send2trash==1.8.0 + # via notebook six==1.16.0 # via # absl-py # azure-core # azure-identity + # bleach # cryptography # google-api-core # google-auth @@ -502,6 +667,7 @@ six==1.16.0 # google-cloud-core # google-resumable-media # grpcio + # isodate # mock # msrestazure # pandavro @@ -541,8 +707,14 @@ tenacity==8.0.1 # via feast (setup.py) tensorflow-metadata==1.6.0 # via feast (setup.py) +termcolor==1.1.0 + # via great-expectations +terminado==0.13.1 + # via notebook testcontainers==3.4.2 # via feast (setup.py) +testpath==0.5.0 + # via nbconvert toml==0.10.2 # via # black @@ -554,40 +726,80 @@ tomli==2.0.0 # coverage # mypy # pep517 +toolz==0.11.2 + # via altair +tornado==6.1 + # via + # ipykernel + # jupyter-client + # notebook + # terminado tqdm==4.62.3 - # via feast (setup.py) -typed-ast==1.5.1 - # via black -types-futures==3.3.7 + # via + # feast (setup.py) + # great-expectations +traitlets==5.1.1 + # via + # ipykernel + # ipython + # ipywidgets + # jupyter-client + # jupyter-core + # matplotlib-inline + # nbclient + # nbconvert + # nbformat + # notebook +typed-ast==1.5.2 + # via + # black + # mypy +types-futures==3.3.8 # via types-protobuf -types-protobuf==3.19.5 +types-protobuf==3.19.7 # via # feast (setup.py) # mypy-protobuf -types-python-dateutil==2.8.8 +types-python-dateutil==2.8.9 # via feast (setup.py) types-pytz==2021.3.4 # via feast (setup.py) -types-pyyaml==6.0.3 +types-pyyaml==6.0.4 # via feast (setup.py) -types-redis==4.1.10 +types-redis==4.1.13 # via feast (setup.py) -types-requests==2.27.7 +types-requests==2.27.8 # via feast (setup.py) -types-setuptools==57.4.7 +types-setuptools==57.4.8 # via feast (setup.py) types-tabulate==0.8.5 # via feast (setup.py) -types-urllib3==1.26.7 +types-urllib3==1.26.8 # via types-requests typing-extensions==4.0.1 # via + # aiohttp + # anyio + # argon2-cffi + # asgiref + # async-timeout + # great-expectations + # h11 + # importlib-metadata + # jsonschema # libcst # mypy # pydantic + # starlette # typing-inspect + # uvicorn + # yarl typing-inspect==0.7.1 # via libcst +tzdata==2021.5 + # via pytz-deprecation-shim +tzlocal==4.1 + # via great-expectations uritemplate==4.1.1 # via google-api-python-client urllib3==1.26.8 @@ -597,7 +809,7 @@ urllib3==1.26.8 # minio # requests # responses -uvicorn[standard]==0.17.0 +uvicorn[standard]==0.17.1 # via feast (setup.py) uvloop==0.16.0 # via uvicorn @@ -605,6 +817,10 @@ virtualenv==20.13.0 # via pre-commit watchgod==0.7 # via uvicorn +wcwidth==0.2.5 + # via prompt-toolkit +webencodings==0.5.1 + # via bleach websocket-client==1.2.3 # via docker websockets==10.1 @@ -613,6 +829,8 @@ werkzeug==2.0.2 # via moto wheel==0.37.1 # via pip-tools +widgetsnbextension==3.5.2 + # via ipywidgets wrapt==1.13.3 # via # deprecated @@ -622,7 +840,10 @@ xmltodict==0.12.0 yarl==1.7.2 # via aiohttp zipp==3.7.0 - # via importlib-resources + # via + # importlib-metadata + # importlib-resources + # pep517 # The following packages are considered to be unsafe in a requirements file: # pip diff --git a/sdk/python/requirements/py3.8-requirements.txt b/sdk/python/requirements/py3.8-requirements.txt index e94fe117b4a..90b42760132 100644 --- a/sdk/python/requirements/py3.8-requirements.txt +++ b/sdk/python/requirements/py3.8-requirements.txt @@ -65,7 +65,7 @@ markupsafe==2.0.1 # via jinja2 mmh3==3.0.0 # via feast (setup.py) -numpy==1.22.1 +numpy==1.21.5 # via # pandas # pandavro diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index 69247a2c7dd..1421d7e3c31 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -20,11 +20,21 @@ aiosignal==1.2.0 # via aiohttp alabaster==0.7.12 # via sphinx +altair==4.2.0 + # via great-expectations anyio==3.5.0 # via starlette appdirs==1.4.4 # via black -asgiref==3.4.1 +appnope==0.1.2 + # via + # ipykernel + # ipython +argon2-cffi==21.3.0 + # via notebook +argon2-cffi-bindings==21.2.0 + # via argon2-cffi +asgiref==3.5.0 # via uvicorn asn1crypto==1.4.0 # via @@ -34,6 +44,8 @@ assertpy==1.1 # via feast (setup.py) async-timeout==4.0.2 # via aiohttp +asynctest==0.13.0 + # via aiohttp attrs==21.4.0 # via # aiohttp @@ -55,13 +67,21 @@ azure-storage-blob==12.9.0 # via adlfs babel==2.9.1 # via sphinx +backcall==0.2.0 + # via ipython +backports.zoneinfo==0.2.1 + # via + # pytz-deprecation-shim + # tzlocal black==19.10b0 # via feast (setup.py) -boto3==1.20.40 +bleach==4.1.0 + # via nbconvert +boto3==1.20.46 # via # feast (setup.py) # moto -botocore==1.23.40 +botocore==1.23.46 # via # boto3 # moto @@ -78,12 +98,13 @@ certifi==2021.10.8 # snowflake-connector-python cffi==1.15.0 # via + # argon2-cffi-bindings # azure-datalake-store # cryptography # snowflake-connector-python cfgv==3.3.1 # via pre-commit -charset-normalizer==2.0.10 +charset-normalizer==2.0.11 # via # aiohttp # requests @@ -92,11 +113,12 @@ click==8.0.3 # via # black # feast (setup.py) + # great-expectations # pip-tools # uvicorn colorama==0.4.4 # via feast (setup.py) -coverage[toml]==6.2 +coverage[toml]==6.3 # via pytest-cov cryptography==3.3.2 # via @@ -106,10 +128,17 @@ cryptography==3.3.2 # feast (setup.py) # moto # msal + # pyjwt # pyopenssl # snowflake-connector-python +debugpy==1.5.1 + # via ipykernel decorator==5.1.1 - # via gcsfs + # via + # gcsfs + # ipython +defusedxml==0.7.1 + # via nbconvert deprecated==1.2.13 # via redis deprecation==2.1.0 @@ -126,9 +155,14 @@ docutils==0.17.1 # via # sphinx # sphinx-rtd-theme +entrypoints==0.3 + # via + # altair + # jupyter-client + # nbconvert execnet==1.9.0 # via pytest-xdist -fastapi==0.72.0 +fastapi==0.73.0 # via feast (setup.py) fastavro==1.4.9 # via @@ -206,6 +240,8 @@ googleapis-common-protos==1.52.0 # feast (setup.py) # google-api-core # tensorflow-metadata +great-expectations==0.14.4 + # via feast (setup.py) grpcio==1.43.0 # via # feast (setup.py) @@ -230,7 +266,7 @@ httplib2==0.20.2 # google-auth-httplib2 httptools==0.3.0 # via uvicorn -identify==2.4.4 +identify==2.4.7 # via pre-commit idna==3.3 # via @@ -240,24 +276,83 @@ idna==3.3 # yarl imagesize==1.3.0 # via sphinx +importlib-metadata==4.2.0 + # via + # click + # flake8 + # great-expectations + # jsonschema + # moto + # pep517 + # pluggy + # pre-commit + # pytest + # redis + # virtualenv +importlib-resources==5.4.0 + # via jsonschema iniconfig==1.1.1 # via pytest +ipykernel==6.7.0 + # via + # ipywidgets + # notebook +ipython==7.31.1 + # via + # ipykernel + # ipywidgets +ipython-genutils==0.2.0 + # via + # ipywidgets + # nbformat + # notebook +ipywidgets==7.6.5 + # via great-expectations isodate==0.6.1 # via msrest isort==5.10.1 # via feast (setup.py) +jedi==0.18.1 + # via ipython jinja2==3.0.3 # via + # altair # feast (setup.py) + # great-expectations # moto + # nbconvert + # notebook # sphinx jmespath==0.10.0 # via # boto3 # botocore +jsonpatch==1.32 + # via great-expectations +jsonpointer==2.2 + # via jsonpatch jsonschema==4.4.0 - # via feast (setup.py) -libcst==0.4.0 + # via + # altair + # feast (setup.py) + # great-expectations + # nbformat +jupyter-client==7.1.2 + # via + # ipykernel + # nbclient + # notebook +jupyter-core==4.9.1 + # via + # jupyter-client + # nbconvert + # nbformat + # notebook +jupyterlab-pygments==0.1.2 + # via nbconvert +jupyterlab-widgets==1.0.2 + # via ipywidgets +libcst==0.4.1 # via # google-cloud-bigquery-storage # google-cloud-datastore @@ -265,15 +360,23 @@ markupsafe==2.0.1 # via # jinja2 # moto +matplotlib-inline==0.1.3 + # via + # ipykernel + # ipython mccabe==0.6.1 # via flake8 minio==7.1.0 # via feast (setup.py) +mistune==0.8.4 + # via + # great-expectations + # nbconvert mmh3==3.0.0 # via feast (setup.py) mock==2.0.0 # via feast (setup.py) -moto==3.0.0 +moto==3.0.2 # via feast (setup.py) msal==1.16.0 # via @@ -289,7 +392,7 @@ msrest==0.6.21 # msrestazure msrestazure==0.6.4 # via adlfs -multidict==5.2.0 +multidict==6.0.2 # via # aiohttp # yarl @@ -301,19 +404,41 @@ mypy-extensions==0.4.3 # typing-inspect mypy-protobuf==3.1.0 # via feast (setup.py) +nbclient==0.5.10 + # via nbconvert +nbconvert==6.4.1 + # via notebook +nbformat==5.1.3 + # via + # ipywidgets + # nbclient + # nbconvert + # notebook +nest-asyncio==1.5.4 + # via + # ipykernel + # jupyter-client + # nbclient + # notebook nodeenv==1.6.0 # via pre-commit -numpy==1.22.1 +notebook==6.4.8 + # via widgetsnbextension +numpy==1.21.5 # via + # altair + # great-expectations # pandas # pandavro # pyarrow -oauthlib==3.1.1 + # scipy +oauthlib==3.2.0 # via requests-oauthlib oscrypto==1.2.1 # via snowflake-connector-python packaging==21.3 # via + # bleach # deprecation # google-api-core # google-cloud-bigquery @@ -323,17 +448,27 @@ packaging==21.3 # sphinx pandas==1.3.5 # via + # altair # feast (setup.py) + # great-expectations # pandavro # snowflake-connector-python pandavro==1.5.2 # via feast (setup.py) +pandocfilters==1.5.0 + # via nbconvert +parso==0.8.3 + # via jedi pathspec==0.9.0 # via black pbr==5.8.0 # via mock pep517==0.12.0 # via pip-tools +pexpect==4.8.0 + # via ipython +pickleshare==0.7.5 + # via ipython pip-tools==6.4.0 # via feast (setup.py) platformdirs==2.4.1 @@ -344,6 +479,10 @@ portalocker==2.3.2 # via msal-extensions pre-commit==2.17.0 # via feast (setup.py) +prometheus-client==0.13.1 + # via notebook +prompt-toolkit==3.0.26 + # via ipython proto-plus==1.19.6 # via # feast (setup.py) @@ -351,7 +490,7 @@ proto-plus==1.19.6 # google-cloud-bigquery-storage # google-cloud-datastore # google-cloud-firestore -protobuf==3.19.3 +protobuf==3.19.4 # via # feast (setup.py) # google-api-core @@ -363,6 +502,10 @@ protobuf==3.19.3 # mypy-protobuf # proto-plus # tensorflow-metadata +ptyprocess==0.7.0 + # via + # pexpect + # terminado py==1.11.0 # via # pytest @@ -383,7 +526,7 @@ pycodestyle==2.8.0 # via flake8 pycparser==2.21 # via cffi -pycryptodomex==3.13.0 +pycryptodomex==3.14.0 # via snowflake-connector-python pydantic==1.9.0 # via @@ -392,7 +535,11 @@ pydantic==1.9.0 pyflakes==2.4.0 # via flake8 pygments==2.11.2 - # via sphinx + # via + # ipython + # jupyterlab-pygments + # nbconvert + # sphinx pyjwt[crypto]==2.3.0 # via # adal @@ -400,8 +547,9 @@ pyjwt[crypto]==2.3.0 # snowflake-connector-python pyopenssl==21.0.0 # via snowflake-connector-python -pyparsing==3.0.7 +pyparsing==2.4.7 # via + # great-expectations # httplib2 # packaging pyrsistent==0.18.1 @@ -438,6 +586,8 @@ python-dateutil==2.8.2 # adal # botocore # google-cloud-bigquery + # great-expectations + # jupyter-client # moto # pandas python-dotenv==0.19.2 @@ -446,16 +596,23 @@ pytz==2021.3 # via # babel # google-api-core + # great-expectations # moto # pandas # snowflake-connector-python +pytz-deprecation-shim==0.1.0.post0 + # via tzlocal pyyaml==6.0 # via # feast (setup.py) # libcst # pre-commit # uvicorn -redis==4.1.1 +pyzmq==22.3.0 + # via + # jupyter-client + # notebook +redis==4.1.2 # via feast (setup.py) regex==2022.1.18 # via black @@ -471,6 +628,7 @@ requests==2.27.1 # google-api-core # google-cloud-bigquery # google-cloud-storage + # great-expectations # moto # msal # msrest @@ -478,7 +636,7 @@ requests==2.27.1 # responses # snowflake-connector-python # sphinx -requests-oauthlib==1.3.0 +requests-oauthlib==1.3.1 # via # google-auth-oauthlib # msrest @@ -486,13 +644,22 @@ responses==0.17.0 # via moto rsa==4.8 # via google-auth +ruamel.yaml==0.17.17 + # via great-expectations +ruamel.yaml.clib==0.2.6 + # via ruamel.yaml s3transfer==0.5.0 # via boto3 +scipy==1.7.3 + # via great-expectations +send2trash==1.8.0 + # via notebook six==1.16.0 # via # absl-py # azure-core # azure-identity + # bleach # cryptography # google-api-core # google-auth @@ -500,6 +667,7 @@ six==1.16.0 # google-cloud-core # google-resumable-media # grpcio + # isodate # mock # msrestazure # pandavro @@ -539,8 +707,14 @@ tenacity==8.0.1 # via feast (setup.py) tensorflow-metadata==1.6.0 # via feast (setup.py) +termcolor==1.1.0 + # via great-expectations +terminado==0.13.1 + # via notebook testcontainers==3.4.2 # via feast (setup.py) +testpath==0.5.0 + # via nbconvert toml==0.10.2 # via # black @@ -552,40 +726,80 @@ tomli==2.0.0 # coverage # mypy # pep517 +toolz==0.11.2 + # via altair +tornado==6.1 + # via + # ipykernel + # jupyter-client + # notebook + # terminado tqdm==4.62.3 - # via feast (setup.py) -typed-ast==1.5.1 - # via black -types-futures==3.3.7 + # via + # feast (setup.py) + # great-expectations +traitlets==5.1.1 + # via + # ipykernel + # ipython + # ipywidgets + # jupyter-client + # jupyter-core + # matplotlib-inline + # nbclient + # nbconvert + # nbformat + # notebook +typed-ast==1.5.2 + # via + # black + # mypy +types-futures==3.3.8 # via types-protobuf -types-protobuf==3.19.5 +types-protobuf==3.19.7 # via # feast (setup.py) # mypy-protobuf -types-python-dateutil==2.8.8 +types-python-dateutil==2.8.9 # via feast (setup.py) types-pytz==2021.3.4 # via feast (setup.py) -types-pyyaml==6.0.3 +types-pyyaml==6.0.4 # via feast (setup.py) -types-redis==4.1.10 +types-redis==4.1.13 # via feast (setup.py) -types-requests==2.27.7 +types-requests==2.27.8 # via feast (setup.py) -types-setuptools==57.4.7 +types-setuptools==57.4.8 # via feast (setup.py) types-tabulate==0.8.5 # via feast (setup.py) -types-urllib3==1.26.7 +types-urllib3==1.26.8 # via types-requests typing-extensions==4.0.1 # via + # aiohttp + # anyio + # argon2-cffi + # asgiref + # async-timeout + # great-expectations + # h11 + # importlib-metadata + # jsonschema # libcst # mypy # pydantic + # starlette # typing-inspect + # uvicorn + # yarl typing-inspect==0.7.1 # via libcst +tzdata==2021.5 + # via pytz-deprecation-shim +tzlocal==4.1 + # via great-expectations uritemplate==4.1.1 # via google-api-python-client urllib3==1.26.8 @@ -595,7 +809,7 @@ urllib3==1.26.8 # minio # requests # responses -uvicorn[standard]==0.17.0 +uvicorn[standard]==0.17.1 # via feast (setup.py) uvloop==0.16.0 # via uvicorn @@ -603,6 +817,10 @@ virtualenv==20.13.0 # via pre-commit watchgod==0.7 # via uvicorn +wcwidth==0.2.5 + # via prompt-toolkit +webencodings==0.5.1 + # via bleach websocket-client==1.2.3 # via docker websockets==10.1 @@ -611,6 +829,8 @@ werkzeug==2.0.2 # via moto wheel==0.37.1 # via pip-tools +widgetsnbextension==3.5.2 + # via ipywidgets wrapt==1.13.3 # via # deprecated @@ -619,6 +839,11 @@ xmltodict==0.12.0 # via moto yarl==1.7.2 # via aiohttp +zipp==3.7.0 + # via + # importlib-metadata + # importlib-resources + # pep517 # The following packages are considered to be unsafe in a requirements file: # pip diff --git a/sdk/python/requirements/py3.9-requirements.txt b/sdk/python/requirements/py3.9-requirements.txt index 187cb02154b..8db9fd4b14f 100644 --- a/sdk/python/requirements/py3.9-requirements.txt +++ b/sdk/python/requirements/py3.9-requirements.txt @@ -63,7 +63,7 @@ markupsafe==2.0.1 # via jinja2 mmh3==3.0.0 # via feast (setup.py) -numpy==1.22.1 +numpy==1.21.5 # via # pandas # pandavro diff --git a/sdk/python/setup.py b/sdk/python/setup.py index cb5381813b5..7535987f833 100644 --- a/sdk/python/setup.py +++ b/sdk/python/setup.py @@ -90,8 +90,12 @@ "snowflake-connector-python[pandas]>=2.7.3", ] +GE_REQUIRED = [ + "great_expectations>=0.14.0,<0.15.0" +] + CI_REQUIRED = ( - [ + [ "cryptography==3.3.2", "flake8", "black==19.10b0", @@ -131,10 +135,11 @@ "types-setuptools", "types-tabulate", ] - + GCP_REQUIRED - + REDIS_REQUIRED - + AWS_REQUIRED - + SNOWFLAKE_REQUIRED + + GCP_REQUIRED + + REDIS_REQUIRED + + AWS_REQUIRED + + SNOWFLAKE_REQUIRED + + GE_REQUIRED ) DEV_REQUIRED = ["mypy-protobuf>=3.1.0", "grpcio-testing==1.*"] + CI_REQUIRED @@ -236,7 +241,8 @@ def run(self): "gcp": GCP_REQUIRED, "aws": AWS_REQUIRED, "redis": REDIS_REQUIRED, - "snowflake": SNOWFLAKE_REQUIRED + "snowflake": SNOWFLAKE_REQUIRED, + "ge": GE_REQUIRED, }, include_package_data=True, license="Apache", diff --git a/sdk/python/tests/__init__.py b/sdk/python/tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/sdk/python/tests/data/__init__.py b/sdk/python/tests/data/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/sdk/python/tests/integration/e2e/test_validation.py b/sdk/python/tests/integration/e2e/test_validation.py new file mode 100644 index 00000000000..2bd1e3cbbc9 --- /dev/null +++ b/sdk/python/tests/integration/e2e/test_validation.py @@ -0,0 +1,134 @@ +import pandas as pd +import pytest +from great_expectations.core import ExpectationSuite +from great_expectations.dataset import PandasDataset + +from feast.dqm.errors import ValidationFailed +from feast.dqm.profilers.ge_profiler import ge_profiler +from tests.integration.feature_repos.repo_configuration import ( + construct_universal_feature_views, +) +from tests.integration.feature_repos.universal.entities import ( + customer, + driver, + location, +) + +_features = [ + "customer_profile:current_balance", + "customer_profile:avg_passenger_count", + "customer_profile:lifetime_trip_count", + "order:order_is_success", + "global_stats:num_rides", + "global_stats:avg_ride_length", +] + + +@ge_profiler +def configurable_profiler(dataset: PandasDataset) -> ExpectationSuite: + from great_expectations.profile.user_configurable_profiler import ( + UserConfigurableProfiler, + ) + + return UserConfigurableProfiler( + profile_dataset=dataset, + excluded_expectations=[ + "expect_table_columns_to_match_ordered_list", + "expect_table_row_count_to_be_between", + ], + value_set_threshold="few", + ).build_suite() + + +@ge_profiler +def profiler_with_unrealistic_expectations(dataset: PandasDataset) -> ExpectationSuite: + # need to create dataframe with corrupted data first + df = pd.DataFrame() + df["current_balance"] = [-100] + df["avg_passenger_count"] = [0] + + other_ds = PandasDataset(df) + other_ds.expect_column_max_to_be_between("current_balance", -1000, -100) + other_ds.expect_column_values_to_be_in_set("avg_passenger_count", value_set={0}) + + # this should pass + other_ds.expect_column_min_to_be_between("avg_passenger_count", 0, 1000) + + return other_ds.get_expectation_suite() + + +@pytest.mark.integration +@pytest.mark.universal +def test_historical_retrieval_with_validation(environment, universal_data_sources): + store = environment.feature_store + + (entities, datasets, data_sources) = universal_data_sources + feature_views = construct_universal_feature_views(data_sources) + + store.apply([driver(), customer(), location(), *feature_views.values()]) + + entity_df = datasets["entity"].drop( + columns=["order_id", "origin_id", "destination_id"] + ) + + reference_job = store.get_historical_features( + entity_df=entity_df, features=_features, + ) + + store.create_saved_dataset( + from_=reference_job, + name="my_training_dataset", + storage=environment.data_source_creator.create_saved_dataset_destination(), + ) + + job = store.get_historical_features(entity_df=entity_df, features=_features,) + + # if validation pass there will be no exceptions on this point + job.to_df( + validation_reference=store.get_saved_dataset( + "my_training_dataset" + ).as_reference(profiler=configurable_profiler) + ) + + +@pytest.mark.integration +@pytest.mark.universal +def test_historical_retrieval_fails_on_validation(environment, universal_data_sources): + store = environment.feature_store + + (entities, datasets, data_sources) = universal_data_sources + feature_views = construct_universal_feature_views(data_sources) + + store.apply([driver(), customer(), location(), *feature_views.values()]) + + entity_df = datasets["entity"].drop( + columns=["order_id", "origin_id", "destination_id"] + ) + + reference_job = store.get_historical_features( + entity_df=entity_df, features=_features, + ) + + store.create_saved_dataset( + from_=reference_job, + name="my_other_dataset", + storage=environment.data_source_creator.create_saved_dataset_destination(), + ) + + job = store.get_historical_features(entity_df=entity_df, features=_features,) + + with pytest.raises(ValidationFailed) as exc_info: + job.to_df( + validation_reference=store.get_saved_dataset( + "my_other_dataset" + ).as_reference(profiler=profiler_with_unrealistic_expectations) + ) + + failed_expectations = exc_info.value.report.errors + assert len(failed_expectations) == 2 + + assert failed_expectations[0].check_name == "expect_column_max_to_be_between" + assert failed_expectations[0].column_name == "current_balance" + + assert failed_expectations[1].check_name == "expect_column_values_to_be_in_set" + assert failed_expectations[1].column_name == "avg_passenger_count"