From 1a91a330f8f33dfdb8637aecd30b16cb532cb901 Mon Sep 17 00:00:00 2001 From: Cody Lin Date: Mon, 7 Jun 2021 16:58:54 -0700 Subject: [PATCH 01/20] Initial commit to catch nonexistent table Signed-off-by: Cody Lin Signed-off-by: Cody Lin --- sdk/python/feast/data_source.py | 18 ++++++++++++++++++ sdk/python/tests/test_data_source.py | 20 ++++++++++++++++++++ sdk/python/tests/utils/data_source_utils.py | 6 ++++++ 3 files changed, 44 insertions(+) create mode 100644 sdk/python/tests/test_data_source.py diff --git a/sdk/python/feast/data_source.py b/sdk/python/feast/data_source.py index 44badcb83b6..d3e51634759 100644 --- a/sdk/python/feast/data_source.py +++ b/sdk/python/feast/data_source.py @@ -636,6 +636,13 @@ def __init__( ): self._bigquery_options = BigQueryOptions(table_ref=table_ref, query=query) + if not self._table_exists(): + raise NameError( + f""" + Unable to find table {table_ref} in BigQuery. Please check that table exists. + """ + ) + super().__init__( event_timestamp_column, created_timestamp_column, @@ -723,6 +730,17 @@ def get_table_column_names_and_types(self) -> Iterable[Tuple[str, str]]: return name_type_pairs + def _table_exists(self) -> bool: + from google.api_core.exceptions import NotFound + from google.cloud import bigquery + + client = bigquery.Client() + try: + client.get_table(self.table_ref) + return True + except NotFound: + return False + class KafkaSource(DataSource): def __init__( diff --git a/sdk/python/tests/test_data_source.py b/sdk/python/tests/test_data_source.py new file mode 100644 index 00000000000..03940f78ab5 --- /dev/null +++ b/sdk/python/tests/test_data_source.py @@ -0,0 +1,20 @@ +import pytest +from utils.data_source_utils import ( + nonexistent_bq_source, + simple_bq_source_using_table_ref_arg, +) + +from feast import Entity, ValueType +from feast.feature_view import FeatureView +from feast.inference import infer_entity_value_type_from_feature_views + + +@pytest.mark.integration +def test_existent_bq_source(simple_dataset_1): + existent_bq = simple_bq_source_using_table_ref_arg(simple_dataset_1) + + +@pytest.mark.integration +def test_nonexistent_bq_source(): + with pytest.raises(NameError): + nonexistent_bq = nonexistent_bq_source() diff --git a/sdk/python/tests/utils/data_source_utils.py b/sdk/python/tests/utils/data_source_utils.py index c848b8ea647..d29418a4540 100644 --- a/sdk/python/tests/utils/data_source_utils.py +++ b/sdk/python/tests/utils/data_source_utils.py @@ -54,3 +54,9 @@ def simple_bq_source_using_query_arg(df, event_timestamp_column=None) -> BigQuer query=f"SELECT * FROM {bq_source_using_table_ref.table_ref}", event_timestamp_column=event_timestamp_column, ) + + +def nonexistent_bq_source() -> BigQuerySource: + client = bigquery.Client() + table_ref = "project.dataset.nonexistent_table" + return BigQuerySource(table_ref=table_ref, event_timestamp_column="") From da7a71248f22c7ae5279a90efbd1ea050848396f Mon Sep 17 00:00:00 2001 From: Cody Lin Date: Mon, 7 Jun 2021 17:33:02 -0700 Subject: [PATCH 02/20] simplify nonexistent BQ table test Signed-off-by: Cody Lin --- sdk/python/tests/test_data_source.py | 15 ++++++--------- sdk/python/tests/utils/data_source_utils.py | 6 ------ 2 files changed, 6 insertions(+), 15 deletions(-) diff --git a/sdk/python/tests/test_data_source.py b/sdk/python/tests/test_data_source.py index 03940f78ab5..0da214135f7 100644 --- a/sdk/python/tests/test_data_source.py +++ b/sdk/python/tests/test_data_source.py @@ -1,12 +1,6 @@ import pytest -from utils.data_source_utils import ( - nonexistent_bq_source, - simple_bq_source_using_table_ref_arg, -) - -from feast import Entity, ValueType -from feast.feature_view import FeatureView -from feast.inference import infer_entity_value_type_from_feature_views +from google.cloud import bigquery +from utils.data_source_utils import simple_bq_source_using_table_ref_arg @pytest.mark.integration @@ -16,5 +10,8 @@ def test_existent_bq_source(simple_dataset_1): @pytest.mark.integration def test_nonexistent_bq_source(): + client = bigquery.Client() + table_ref = "project.dataset.nonexistent_table" + with pytest.raises(NameError): - nonexistent_bq = nonexistent_bq_source() + nonexistent_bq = BigQuerySource(table_ref=table_ref, event_timestamp_column="") diff --git a/sdk/python/tests/utils/data_source_utils.py b/sdk/python/tests/utils/data_source_utils.py index d29418a4540..c848b8ea647 100644 --- a/sdk/python/tests/utils/data_source_utils.py +++ b/sdk/python/tests/utils/data_source_utils.py @@ -54,9 +54,3 @@ def simple_bq_source_using_query_arg(df, event_timestamp_column=None) -> BigQuer query=f"SELECT * FROM {bq_source_using_table_ref.table_ref}", event_timestamp_column=event_timestamp_column, ) - - -def nonexistent_bq_source() -> BigQuerySource: - client = bigquery.Client() - table_ref = "project.dataset.nonexistent_table" - return BigQuerySource(table_ref=table_ref, event_timestamp_column="") From f6a2f98027d05cd9cc9495d5eba3cd273db15a7a Mon Sep 17 00:00:00 2001 From: Cody Lin Date: Tue, 8 Jun 2021 08:54:14 -0700 Subject: [PATCH 03/20] clean up table_exists exception Signed-off-by: Cody Lin --- sdk/python/feast/data_source.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/sdk/python/feast/data_source.py b/sdk/python/feast/data_source.py index d3e51634759..e886b523f18 100644 --- a/sdk/python/feast/data_source.py +++ b/sdk/python/feast/data_source.py @@ -635,13 +635,7 @@ def __init__( query: Optional[str] = None, ): self._bigquery_options = BigQueryOptions(table_ref=table_ref, query=query) - - if not self._table_exists(): - raise NameError( - f""" - Unable to find table {table_ref} in BigQuery. Please check that table exists. - """ - ) + self._table_exists() super().__init__( event_timestamp_column, @@ -730,16 +724,20 @@ def get_table_column_names_and_types(self) -> Iterable[Tuple[str, str]]: return name_type_pairs - def _table_exists(self) -> bool: + def _table_exists(self): from google.api_core.exceptions import NotFound from google.cloud import bigquery client = bigquery.Client() + try: client.get_table(self.table_ref) - return True except NotFound: - return False + raise NameError( + f""" + Unable to find table {self.table_ref} in BigQuery. Please check that table exists. + """ + ) class KafkaSource(DataSource): From 41e69a79a7662fd30903e2ac236cff0c3e7556fa Mon Sep 17 00:00:00 2001 From: Cody Lin Date: Tue, 8 Jun 2021 10:56:38 -0700 Subject: [PATCH 04/20] remove unneeded variable Signed-off-by: Cody Lin --- sdk/python/tests/test_data_source.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sdk/python/tests/test_data_source.py b/sdk/python/tests/test_data_source.py index 0da214135f7..54788aea441 100644 --- a/sdk/python/tests/test_data_source.py +++ b/sdk/python/tests/test_data_source.py @@ -11,7 +11,8 @@ def test_existent_bq_source(simple_dataset_1): @pytest.mark.integration def test_nonexistent_bq_source(): client = bigquery.Client() - table_ref = "project.dataset.nonexistent_table" with pytest.raises(NameError): - nonexistent_bq = BigQuerySource(table_ref=table_ref, event_timestamp_column="") + nonexistent_bq = BigQuerySource( + table_ref="project.dataset.nonexistent_table", event_timestamp_column="" + ) From af937c4b4c15bc5a5c64cd67727f520d4dd4640e Mon Sep 17 00:00:00 2001 From: Cody Lin Date: Tue, 8 Jun 2021 15:20:23 -0700 Subject: [PATCH 05/20] function name change to _assert_table_exists Signed-off-by: Cody Lin --- sdk/python/feast/data_source.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/python/feast/data_source.py b/sdk/python/feast/data_source.py index e886b523f18..05ffed851b8 100644 --- a/sdk/python/feast/data_source.py +++ b/sdk/python/feast/data_source.py @@ -635,7 +635,7 @@ def __init__( query: Optional[str] = None, ): self._bigquery_options = BigQueryOptions(table_ref=table_ref, query=query) - self._table_exists() + self._assert_table_exists() super().__init__( event_timestamp_column, @@ -724,7 +724,7 @@ def get_table_column_names_and_types(self) -> Iterable[Tuple[str, str]]: return name_type_pairs - def _table_exists(self): + def _assert_table_exists(self): from google.api_core.exceptions import NotFound from google.cloud import bigquery From 215de4bea8707cd084bad7f4d84b13a93f0360fd Mon Sep 17 00:00:00 2001 From: Cody Lin Date: Mon, 7 Jun 2021 16:58:54 -0700 Subject: [PATCH 06/20] Initial commit to catch nonexistent table Signed-off-by: Cody Lin Signed-off-by: Cody Lin --- sdk/python/feast/data_source.py | 19 +++++++++++++++++++ sdk/python/tests/utils/data_source_utils.py | 6 ++++++ 2 files changed, 25 insertions(+) diff --git a/sdk/python/feast/data_source.py b/sdk/python/feast/data_source.py index 05ffed851b8..4aeb3032ecc 100644 --- a/sdk/python/feast/data_source.py +++ b/sdk/python/feast/data_source.py @@ -637,6 +637,13 @@ def __init__( self._bigquery_options = BigQueryOptions(table_ref=table_ref, query=query) self._assert_table_exists() + if not self._table_exists(): + raise NameError( + f""" + Unable to find table {table_ref} in BigQuery. Please check that table exists. + """ + ) + super().__init__( event_timestamp_column, created_timestamp_column, @@ -724,11 +731,16 @@ def get_table_column_names_and_types(self) -> Iterable[Tuple[str, str]]: return name_type_pairs +<<<<<<< HEAD def _assert_table_exists(self): +======= + def _table_exists(self) -> bool: +>>>>>>> f23e9d89... Initial commit to catch nonexistent table from google.api_core.exceptions import NotFound from google.cloud import bigquery client = bigquery.Client() +<<<<<<< HEAD try: client.get_table(self.table_ref) @@ -738,6 +750,13 @@ def _assert_table_exists(self): Unable to find table {self.table_ref} in BigQuery. Please check that table exists. """ ) +======= + try: + client.get_table(self.table_ref) + return True + except NotFound: + return False +>>>>>>> f23e9d89... Initial commit to catch nonexistent table class KafkaSource(DataSource): diff --git a/sdk/python/tests/utils/data_source_utils.py b/sdk/python/tests/utils/data_source_utils.py index c848b8ea647..d29418a4540 100644 --- a/sdk/python/tests/utils/data_source_utils.py +++ b/sdk/python/tests/utils/data_source_utils.py @@ -54,3 +54,9 @@ def simple_bq_source_using_query_arg(df, event_timestamp_column=None) -> BigQuer query=f"SELECT * FROM {bq_source_using_table_ref.table_ref}", event_timestamp_column=event_timestamp_column, ) + + +def nonexistent_bq_source() -> BigQuerySource: + client = bigquery.Client() + table_ref = "project.dataset.nonexistent_table" + return BigQuerySource(table_ref=table_ref, event_timestamp_column="") From caac9d0e43cc7f34de584db7fef0f6df8b598d74 Mon Sep 17 00:00:00 2001 From: Cody Lin Date: Mon, 7 Jun 2021 17:33:02 -0700 Subject: [PATCH 07/20] simplify nonexistent BQ table test Signed-off-by: Cody Lin --- sdk/python/tests/utils/data_source_utils.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/sdk/python/tests/utils/data_source_utils.py b/sdk/python/tests/utils/data_source_utils.py index d29418a4540..c848b8ea647 100644 --- a/sdk/python/tests/utils/data_source_utils.py +++ b/sdk/python/tests/utils/data_source_utils.py @@ -54,9 +54,3 @@ def simple_bq_source_using_query_arg(df, event_timestamp_column=None) -> BigQuer query=f"SELECT * FROM {bq_source_using_table_ref.table_ref}", event_timestamp_column=event_timestamp_column, ) - - -def nonexistent_bq_source() -> BigQuerySource: - client = bigquery.Client() - table_ref = "project.dataset.nonexistent_table" - return BigQuerySource(table_ref=table_ref, event_timestamp_column="") From 2d2a484ec23878eb241a4dea4c47604f05439a8b Mon Sep 17 00:00:00 2001 From: Cody Lin Date: Tue, 8 Jun 2021 08:54:14 -0700 Subject: [PATCH 08/20] clean up table_exists exception Signed-off-by: Cody Lin --- sdk/python/feast/data_source.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/sdk/python/feast/data_source.py b/sdk/python/feast/data_source.py index 4aeb3032ecc..f7871a1c610 100644 --- a/sdk/python/feast/data_source.py +++ b/sdk/python/feast/data_source.py @@ -635,6 +635,7 @@ def __init__( query: Optional[str] = None, ): self._bigquery_options = BigQueryOptions(table_ref=table_ref, query=query) +<<<<<<< HEAD self._assert_table_exists() if not self._table_exists(): @@ -643,6 +644,9 @@ def __init__( Unable to find table {table_ref} in BigQuery. Please check that table exists. """ ) +======= + self._table_exists() +>>>>>>> c6935e99... clean up table_exists exception super().__init__( event_timestamp_column, @@ -731,16 +735,11 @@ def get_table_column_names_and_types(self) -> Iterable[Tuple[str, str]]: return name_type_pairs -<<<<<<< HEAD def _assert_table_exists(self): -======= - def _table_exists(self) -> bool: ->>>>>>> f23e9d89... Initial commit to catch nonexistent table from google.api_core.exceptions import NotFound from google.cloud import bigquery client = bigquery.Client() -<<<<<<< HEAD try: client.get_table(self.table_ref) @@ -750,13 +749,6 @@ def _table_exists(self) -> bool: Unable to find table {self.table_ref} in BigQuery. Please check that table exists. """ ) -======= - try: - client.get_table(self.table_ref) - return True - except NotFound: - return False ->>>>>>> f23e9d89... Initial commit to catch nonexistent table class KafkaSource(DataSource): From c548ab6481fc432c52cf85c61de6d19fbc23564d Mon Sep 17 00:00:00 2001 From: Cody Lin Date: Tue, 8 Jun 2021 15:20:23 -0700 Subject: [PATCH 09/20] function name change to _assert_table_exists Signed-off-by: Cody Lin --- sdk/python/feast/data_source.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/sdk/python/feast/data_source.py b/sdk/python/feast/data_source.py index f7871a1c610..05ffed851b8 100644 --- a/sdk/python/feast/data_source.py +++ b/sdk/python/feast/data_source.py @@ -635,19 +635,8 @@ def __init__( query: Optional[str] = None, ): self._bigquery_options = BigQueryOptions(table_ref=table_ref, query=query) -<<<<<<< HEAD self._assert_table_exists() - if not self._table_exists(): - raise NameError( - f""" - Unable to find table {table_ref} in BigQuery. Please check that table exists. - """ - ) -======= - self._table_exists() ->>>>>>> c6935e99... clean up table_exists exception - super().__init__( event_timestamp_column, created_timestamp_column, From d5eb8af667a779a48572cc0aa5f17494ce289219 Mon Sep 17 00:00:00 2001 From: Cody Lin Date: Wed, 9 Jun 2021 11:19:40 -0700 Subject: [PATCH 10/20] fix lint errors and rebase Signed-off-by: Cody Lin --- sdk/python/tests/test_data_source.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/sdk/python/tests/test_data_source.py b/sdk/python/tests/test_data_source.py index 54788aea441..076a72043d2 100644 --- a/sdk/python/tests/test_data_source.py +++ b/sdk/python/tests/test_data_source.py @@ -1,18 +1,17 @@ import pytest -from google.cloud import bigquery from utils.data_source_utils import simple_bq_source_using_table_ref_arg +from feast.data_source import BigQuerySource + @pytest.mark.integration def test_existent_bq_source(simple_dataset_1): - existent_bq = simple_bq_source_using_table_ref_arg(simple_dataset_1) + simple_bq_source_using_table_ref_arg(simple_dataset_1) @pytest.mark.integration def test_nonexistent_bq_source(): - client = bigquery.Client() - with pytest.raises(NameError): - nonexistent_bq = BigQuerySource( + BigQuerySource( table_ref="project.dataset.nonexistent_table", event_timestamp_column="" ) From 21a7631f4d9499e3bcc49526f62131d395f52a3a Mon Sep 17 00:00:00 2001 From: Cody Lin Date: Wed, 9 Jun 2021 14:55:44 -0700 Subject: [PATCH 11/20] Fix get_table(None) error Signed-off-by: Cody Lin --- sdk/python/feast/data_source.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sdk/python/feast/data_source.py b/sdk/python/feast/data_source.py index 05ffed851b8..da09b6adf87 100644 --- a/sdk/python/feast/data_source.py +++ b/sdk/python/feast/data_source.py @@ -635,7 +635,8 @@ def __init__( query: Optional[str] = None, ): self._bigquery_options = BigQueryOptions(table_ref=table_ref, query=query) - self._assert_table_exists() + if self.table_ref: + self._assert_table_exists() super().__init__( event_timestamp_column, From 50831ef692421c7bba40efca2bc8ec21a8d4c8d0 Mon Sep 17 00:00:00 2001 From: Cody Lin Date: Wed, 9 Jun 2021 16:36:48 -0700 Subject: [PATCH 12/20] custom exception for both missing file and BQ source Signed-off-by: Cody Lin --- sdk/python/feast/data_source.py | 14 +++++++++----- sdk/python/feast/errors.py | 16 ++++++++++++++++ sdk/python/tests/test_data_source.py | 19 ++++++++++++++++--- 3 files changed, 41 insertions(+), 8 deletions(-) diff --git a/sdk/python/feast/data_source.py b/sdk/python/feast/data_source.py index da09b6adf87..313ca88457c 100644 --- a/sdk/python/feast/data_source.py +++ b/sdk/python/feast/data_source.py @@ -20,6 +20,7 @@ from feast import type_map from feast.data_format import FileFormat, StreamFormat +from feast.errors import BigQuerySourceNotFoundException, FileSourceNotFoundException from feast.protos.feast.core.DataSource_pb2 import DataSource as DataSourceProto from feast.value_type import ValueType @@ -561,6 +562,7 @@ def __init__( file_url = path self._file_options = FileOptions(file_format=file_format, file_url=file_url) + self._assert_table_exists() super().__init__( event_timestamp_column, @@ -623,6 +625,12 @@ def get_table_column_names_and_types(self) -> Iterable[Tuple[str, str]]: schema = ParquetFile(self.path).schema_arrow return zip(schema.names, map(str, schema.types)) + def _assert_table_exists(self): + try: + ParquetFile(self.path).schema_arrow + except (FileNotFoundError): + raise FileSourceNotFoundException(self.path) + class BigQuerySource(DataSource): def __init__( @@ -734,11 +742,7 @@ def _assert_table_exists(self): try: client.get_table(self.table_ref) except NotFound: - raise NameError( - f""" - Unable to find table {self.table_ref} in BigQuery. Please check that table exists. - """ - ) + raise BigQuerySourceNotFoundException(self.table_ref) class KafkaSource(DataSource): diff --git a/sdk/python/feast/errors.py b/sdk/python/feast/errors.py index 5bf9b27553c..214f510faf9 100644 --- a/sdk/python/feast/errors.py +++ b/sdk/python/feast/errors.py @@ -3,6 +3,22 @@ from colorama import Fore, Style +class DataSourceNotFoundException(Exception): + pass + + +class BigQuerySourceNotFoundException(DataSourceNotFoundException): + def __init__(self, table_ref): + super().__init__( + f"Unable to find table '{table_ref}' in BigQuery. Please check that table exists." + ) + + +class FileSourceNotFoundException(DataSourceNotFoundException): + def __init__(self, path): + super().__init__(f"Unable to find table at '{path}'") + + class FeastObjectNotFoundException(Exception): pass diff --git a/sdk/python/tests/test_data_source.py b/sdk/python/tests/test_data_source.py index 076a72043d2..575d13941e8 100644 --- a/sdk/python/tests/test_data_source.py +++ b/sdk/python/tests/test_data_source.py @@ -1,7 +1,20 @@ import pytest -from utils.data_source_utils import simple_bq_source_using_table_ref_arg +from utils.data_source_utils import ( + prep_file_source, + simple_bq_source_using_table_ref_arg, +) -from feast.data_source import BigQuerySource +from feast.data_source import BigQuerySource, FileSource +from feast.errors import BigQuerySourceNotFoundException, FileSourceNotFoundException + + +def test_existent_file_source(simple_dataset_1): + prep_file_source(df=simple_dataset_1) + + +def test_nonexistent_file_source(simple_dataset_1): + with pytest.raises(FileSourceNotFoundException): + FileSource(file_url="nonexistent_file") @pytest.mark.integration @@ -11,7 +24,7 @@ def test_existent_bq_source(simple_dataset_1): @pytest.mark.integration def test_nonexistent_bq_source(): - with pytest.raises(NameError): + with pytest.raises(BigQuerySourceNotFoundException): BigQuerySource( table_ref="project.dataset.nonexistent_table", event_timestamp_column="" ) From 4e7a803b04a1eb51c08567dd0bb206545596cded Mon Sep 17 00:00:00 2001 From: Cody Lin Date: Thu, 10 Jun 2021 08:52:50 -0700 Subject: [PATCH 13/20] revert FileSource checks Signed-off-by: Cody Lin --- sdk/python/feast/data_source.py | 15 ++++----------- sdk/python/feast/errors.py | 5 ----- sdk/python/tests/test_data_source.py | 9 ++------- 3 files changed, 6 insertions(+), 23 deletions(-) diff --git a/sdk/python/feast/data_source.py b/sdk/python/feast/data_source.py index 313ca88457c..940d80adf2a 100644 --- a/sdk/python/feast/data_source.py +++ b/sdk/python/feast/data_source.py @@ -20,7 +20,7 @@ from feast import type_map from feast.data_format import FileFormat, StreamFormat -from feast.errors import BigQuerySourceNotFoundException, FileSourceNotFoundException +from feast.errors import BigQuerySourceNotFoundException from feast.protos.feast.core.DataSource_pb2 import DataSource as DataSourceProto from feast.value_type import ValueType @@ -562,7 +562,6 @@ def __init__( file_url = path self._file_options = FileOptions(file_format=file_format, file_url=file_url) - self._assert_table_exists() super().__init__( event_timestamp_column, @@ -625,12 +624,6 @@ def get_table_column_names_and_types(self) -> Iterable[Tuple[str, str]]: schema = ParquetFile(self.path).schema_arrow return zip(schema.names, map(str, schema.types)) - def _assert_table_exists(self): - try: - ParquetFile(self.path).schema_arrow - except (FileNotFoundError): - raise FileSourceNotFoundException(self.path) - class BigQuerySource(DataSource): def __init__( @@ -643,8 +636,7 @@ def __init__( query: Optional[str] = None, ): self._bigquery_options = BigQueryOptions(table_ref=table_ref, query=query) - if self.table_ref: - self._assert_table_exists() + self._assert_table_exists() super().__init__( event_timestamp_column, @@ -738,7 +730,8 @@ def _assert_table_exists(self): from google.cloud import bigquery client = bigquery.Client() - + if not self.table_ref: + raise BigQuerySourceNotFoundException(self.table_ref) try: client.get_table(self.table_ref) except NotFound: diff --git a/sdk/python/feast/errors.py b/sdk/python/feast/errors.py index 214f510faf9..9ff8de3b890 100644 --- a/sdk/python/feast/errors.py +++ b/sdk/python/feast/errors.py @@ -14,11 +14,6 @@ def __init__(self, table_ref): ) -class FileSourceNotFoundException(DataSourceNotFoundException): - def __init__(self, path): - super().__init__(f"Unable to find table at '{path}'") - - class FeastObjectNotFoundException(Exception): pass diff --git a/sdk/python/tests/test_data_source.py b/sdk/python/tests/test_data_source.py index 575d13941e8..bfe91e34ced 100644 --- a/sdk/python/tests/test_data_source.py +++ b/sdk/python/tests/test_data_source.py @@ -4,19 +4,14 @@ simple_bq_source_using_table_ref_arg, ) -from feast.data_source import BigQuerySource, FileSource -from feast.errors import BigQuerySourceNotFoundException, FileSourceNotFoundException +from feast.data_source import BigQuerySource +from feast.errors import BigQuerySourceNotFoundException def test_existent_file_source(simple_dataset_1): prep_file_source(df=simple_dataset_1) -def test_nonexistent_file_source(simple_dataset_1): - with pytest.raises(FileSourceNotFoundException): - FileSource(file_url="nonexistent_file") - - @pytest.mark.integration def test_existent_bq_source(simple_dataset_1): simple_bq_source_using_table_ref_arg(simple_dataset_1) From 2d99b058165e80950ba8b5d57cb6965dcb7e417e Mon Sep 17 00:00:00 2001 From: Cody Lin Date: Fri, 11 Jun 2021 09:35:14 -0700 Subject: [PATCH 14/20] Use DataSourceNotFoundException instead of subclassing Signed-off-by: Cody Lin --- sdk/python/feast/data_source.py | 6 +++--- sdk/python/feast/errors.py | 8 ++------ sdk/python/tests/test_data_source.py | 4 ++-- 3 files changed, 7 insertions(+), 11 deletions(-) diff --git a/sdk/python/feast/data_source.py b/sdk/python/feast/data_source.py index 940d80adf2a..ceee244d21f 100644 --- a/sdk/python/feast/data_source.py +++ b/sdk/python/feast/data_source.py @@ -20,7 +20,7 @@ from feast import type_map from feast.data_format import FileFormat, StreamFormat -from feast.errors import BigQuerySourceNotFoundException +from feast.errors import DataSourceNotFoundException from feast.protos.feast.core.DataSource_pb2 import DataSource as DataSourceProto from feast.value_type import ValueType @@ -731,11 +731,11 @@ def _assert_table_exists(self): client = bigquery.Client() if not self.table_ref: - raise BigQuerySourceNotFoundException(self.table_ref) + raise DataSourceNotFoundException(self.table_ref) try: client.get_table(self.table_ref) except NotFound: - raise BigQuerySourceNotFoundException(self.table_ref) + raise DataSourceNotFoundException(self.table_ref) class KafkaSource(DataSource): diff --git a/sdk/python/feast/errors.py b/sdk/python/feast/errors.py index 9ff8de3b890..afe9e85ea5c 100644 --- a/sdk/python/feast/errors.py +++ b/sdk/python/feast/errors.py @@ -4,13 +4,9 @@ class DataSourceNotFoundException(Exception): - pass - - -class BigQuerySourceNotFoundException(DataSourceNotFoundException): - def __init__(self, table_ref): + def __init__(self, path): super().__init__( - f"Unable to find table '{table_ref}' in BigQuery. Please check that table exists." + f"Unable to find table at '{path}'. Please check that table exists." ) diff --git a/sdk/python/tests/test_data_source.py b/sdk/python/tests/test_data_source.py index bfe91e34ced..9c8dc034e4c 100644 --- a/sdk/python/tests/test_data_source.py +++ b/sdk/python/tests/test_data_source.py @@ -5,7 +5,7 @@ ) from feast.data_source import BigQuerySource -from feast.errors import BigQuerySourceNotFoundException +from feast.errors import DataSourceNotFoundException def test_existent_file_source(simple_dataset_1): @@ -19,7 +19,7 @@ def test_existent_bq_source(simple_dataset_1): @pytest.mark.integration def test_nonexistent_bq_source(): - with pytest.raises(BigQuerySourceNotFoundException): + with pytest.raises(DataSourceNotFoundException): BigQuerySource( table_ref="project.dataset.nonexistent_table", event_timestamp_column="" ) From c513660407c13480ed9a10099f7fa416122af318 Mon Sep 17 00:00:00 2001 From: Cody Lin Date: Mon, 14 Jun 2021 09:32:27 -0700 Subject: [PATCH 15/20] Moved assert_table_exists out of the BQ constructor to apply_total Signed-off-by: Cody Lin --- sdk/python/feast/data_source.py | 14 --------- sdk/python/feast/repo_operations.py | 1 + sdk/python/tests/example_feature_repo_3.py | 20 +++++++++++++ sdk/python/tests/test_cli_gcp.py | 33 ++++++++++++++++++++++ sdk/python/tests/test_data_source.py | 25 ---------------- 5 files changed, 54 insertions(+), 39 deletions(-) create mode 100644 sdk/python/tests/example_feature_repo_3.py delete mode 100644 sdk/python/tests/test_data_source.py diff --git a/sdk/python/feast/data_source.py b/sdk/python/feast/data_source.py index ceee244d21f..44badcb83b6 100644 --- a/sdk/python/feast/data_source.py +++ b/sdk/python/feast/data_source.py @@ -20,7 +20,6 @@ from feast import type_map from feast.data_format import FileFormat, StreamFormat -from feast.errors import DataSourceNotFoundException from feast.protos.feast.core.DataSource_pb2 import DataSource as DataSourceProto from feast.value_type import ValueType @@ -636,7 +635,6 @@ def __init__( query: Optional[str] = None, ): self._bigquery_options = BigQueryOptions(table_ref=table_ref, query=query) - self._assert_table_exists() super().__init__( event_timestamp_column, @@ -725,18 +723,6 @@ def get_table_column_names_and_types(self) -> Iterable[Tuple[str, str]]: return name_type_pairs - def _assert_table_exists(self): - from google.api_core.exceptions import NotFound - from google.cloud import bigquery - - client = bigquery.Client() - if not self.table_ref: - raise DataSourceNotFoundException(self.table_ref) - try: - client.get_table(self.table_ref) - except NotFound: - raise DataSourceNotFoundException(self.table_ref) - class KafkaSource(DataSource): def __init__( diff --git a/sdk/python/feast/repo_operations.py b/sdk/python/feast/repo_operations.py index b3cc7fa0c39..9086e9e357d 100644 --- a/sdk/python/feast/repo_operations.py +++ b/sdk/python/feast/repo_operations.py @@ -12,6 +12,7 @@ from click.exceptions import BadParameter from feast import Entity, FeatureTable +from feast.errors import DataSourceNotFoundException from feast.feature_view import FeatureView from feast.inference import ( infer_entity_value_type_from_feature_views, diff --git a/sdk/python/tests/example_feature_repo_3.py b/sdk/python/tests/example_feature_repo_3.py new file mode 100644 index 00000000000..3d1dc3394b9 --- /dev/null +++ b/sdk/python/tests/example_feature_repo_3.py @@ -0,0 +1,20 @@ +from datetime import timedelta + +from feast import BigQuerySource, Entity, Feature, FeatureView, ValueType + +nonexistent_source = BigQuerySource( + table_ref="project.dataset.nonexistent_table", event_timestamp_column="" +) + +driver = Entity(name="driver", value_type=ValueType.INT64, description="driver id",) + +nonexistent_features = FeatureView( + name="driver_locations", + entities=["driver"], + ttl=timedelta(days=1), + features=[ + Feature(name="lat", dtype=ValueType.FLOAT), + Feature(name="lon", dtype=ValueType.STRING), + ], + input=nonexistent_source, +) diff --git a/sdk/python/tests/test_cli_gcp.py b/sdk/python/tests/test_cli_gcp.py index 486b04b7efb..e147f37f561 100644 --- a/sdk/python/tests/test_cli_gcp.py +++ b/sdk/python/tests/test_cli_gcp.py @@ -53,3 +53,36 @@ def test_basic() -> None: result = runner.run(["teardown"], cwd=repo_path) assert result.returncode == 0 + + +@pytest.mark.integration +def test_fail() -> None: + project_id = "".join( + random.choice(string.ascii_lowercase + string.digits) for _ in range(10) + ) + runner = CliRunner() + with tempfile.TemporaryDirectory() as repo_dir_name, tempfile.TemporaryDirectory() as data_dir_name: + + repo_path = Path(repo_dir_name) + data_path = Path(data_dir_name) + + repo_config = repo_path / "feature_store.yaml" + + repo_config.write_text( + dedent( + f""" + project: {project_id} + registry: {data_path / "registry.db"} + provider: gcp + """ + ) + ) + + repo_example = repo_path / "example.py" + repo_example.write_text( + (Path(__file__).parent / "example_feature_repo_3.py").read_text() + ) + + returncode, output = runner.run_with_output(["apply"], cwd=repo_path) + assert returncode == 1 + assert "DataSourceNotFoundException" in output diff --git a/sdk/python/tests/test_data_source.py b/sdk/python/tests/test_data_source.py deleted file mode 100644 index 9c8dc034e4c..00000000000 --- a/sdk/python/tests/test_data_source.py +++ /dev/null @@ -1,25 +0,0 @@ -import pytest -from utils.data_source_utils import ( - prep_file_source, - simple_bq_source_using_table_ref_arg, -) - -from feast.data_source import BigQuerySource -from feast.errors import DataSourceNotFoundException - - -def test_existent_file_source(simple_dataset_1): - prep_file_source(df=simple_dataset_1) - - -@pytest.mark.integration -def test_existent_bq_source(simple_dataset_1): - simple_bq_source_using_table_ref_arg(simple_dataset_1) - - -@pytest.mark.integration -def test_nonexistent_bq_source(): - with pytest.raises(DataSourceNotFoundException): - BigQuerySource( - table_ref="project.dataset.nonexistent_table", event_timestamp_column="" - ) From 617dde1e3f6c925f90ee6f3527887100bd8d0593 Mon Sep 17 00:00:00 2001 From: Cody Lin Date: Wed, 16 Jun 2021 14:54:46 -0700 Subject: [PATCH 16/20] rename test and test asset Signed-off-by: Cody Lin --- sdk/python/feast/repo_operations.py | 2 +- ...po_3.py => example_feature_repo_with_missing_bq_source.py} | 0 sdk/python/tests/test_cli_gcp.py | 4 ++-- 3 files changed, 3 insertions(+), 3 deletions(-) rename sdk/python/tests/{example_feature_repo_3.py => example_feature_repo_with_missing_bq_source.py} (100%) diff --git a/sdk/python/feast/repo_operations.py b/sdk/python/feast/repo_operations.py index 9086e9e357d..e00c8d33aa3 100644 --- a/sdk/python/feast/repo_operations.py +++ b/sdk/python/feast/repo_operations.py @@ -155,7 +155,7 @@ def apply_total(repo_config: RepoConfig, repo_path: Path): data_sources = [t.input for t in repo.feature_views] - # Make sure the data source used by this feature view is supported by + # Make sure the data source used by this feature view is supported by Feast for data_source in data_sources: assert_offline_store_supports_data_source( repo_config.offline_store, data_source diff --git a/sdk/python/tests/example_feature_repo_3.py b/sdk/python/tests/example_feature_repo_with_missing_bq_source.py similarity index 100% rename from sdk/python/tests/example_feature_repo_3.py rename to sdk/python/tests/example_feature_repo_with_missing_bq_source.py diff --git a/sdk/python/tests/test_cli_gcp.py b/sdk/python/tests/test_cli_gcp.py index e147f37f561..946cf863b2a 100644 --- a/sdk/python/tests/test_cli_gcp.py +++ b/sdk/python/tests/test_cli_gcp.py @@ -56,7 +56,7 @@ def test_basic() -> None: @pytest.mark.integration -def test_fail() -> None: +def test_missing_bq_source_fail() -> None: project_id = "".join( random.choice(string.ascii_lowercase + string.digits) for _ in range(10) ) @@ -80,7 +80,7 @@ def test_fail() -> None: repo_example = repo_path / "example.py" repo_example.write_text( - (Path(__file__).parent / "example_feature_repo_3.py").read_text() + (Path(__file__).parent / "example_feature_repo_with_missing_bq_source.py").read_text() ) returncode, output = runner.run_with_output(["apply"], cwd=repo_path) From 6b3975a24da2cae221d5676680e5d42d6f49cd71 Mon Sep 17 00:00:00 2001 From: Cody Lin Date: Wed, 16 Jun 2021 17:03:16 -0700 Subject: [PATCH 17/20] move validate logic back to data_source Signed-off-by: Cody Lin --- sdk/python/feast/data_source.py | 56 +++++++++++++++++++++++++++++ sdk/python/feast/repo_operations.py | 2 +- sdk/python/tests/test_cli_gcp.py | 6 ++-- 3 files changed, 61 insertions(+), 3 deletions(-) diff --git a/sdk/python/feast/data_source.py b/sdk/python/feast/data_source.py index 44badcb83b6..ccedca37025 100644 --- a/sdk/python/feast/data_source.py +++ b/sdk/python/feast/data_source.py @@ -20,6 +20,7 @@ from feast import type_map from feast.data_format import FileFormat, StreamFormat +from feast.errors import DataSourceNotFoundException from feast.protos.feast.core.DataSource_pb2 import DataSource as DataSourceProto from feast.value_type import ValueType @@ -519,6 +520,46 @@ def to_proto(self) -> DataSourceProto: """ raise NotImplementedError + def validate(self): + """ + Validates the underlying data source. + """ + raise NotImplementedError + + def _infer_event_timestamp_column(self, ts_column_type_regex_pattern): + ERROR_MSG_PREFIX = "Unable to infer DataSource event_timestamp_column" + USER_GUIDANCE = "Please specify event_timestamp_column explicitly." + + if isinstance(self, FileSource) or isinstance(self, BigQuerySource): + event_timestamp_column, matched_flag = None, False + for col_name, col_datatype in self.get_table_column_names_and_types(): + if re.match(ts_column_type_regex_pattern, col_datatype): + if matched_flag: + raise TypeError( + f""" + {ERROR_MSG_PREFIX} due to multiple possible columns satisfying + the criteria. {USER_GUIDANCE} + """ + ) + matched_flag = True + event_timestamp_column = col_name + if matched_flag: + return event_timestamp_column + else: + raise TypeError( + f""" + {ERROR_MSG_PREFIX} due to an absence of columns that satisfy the criteria. + {USER_GUIDANCE} + """ + ) + else: + raise TypeError( + f""" + {ERROR_MSG_PREFIX} because this DataSource currently does not support this inference. + {USER_GUIDANCE} + """ + ) + class FileSource(DataSource): def __init__( @@ -615,6 +656,10 @@ def to_proto(self) -> DataSourceProto: return data_source_proto + def validate(self): + # TODO: validate a FileSource + pass + @staticmethod def source_datatype_to_feast_value_type() -> Callable[[str], ValueType]: return type_map.pa_to_feast_value_type @@ -692,6 +737,17 @@ def to_proto(self) -> DataSourceProto: return data_source_proto + def validate(self): + if not self.query: + from google.api_core.exceptions import NotFound + from google.cloud import bigquery + + client = bigquery.Client() + try: + client.get_table(self.table_ref) + except NotFound: + raise DataSourceNotFoundException(self.table_ref) + def get_table_query_string(self) -> str: """Returns a string that can directly be used to reference this table in SQL""" if self.table_ref: diff --git a/sdk/python/feast/repo_operations.py b/sdk/python/feast/repo_operations.py index e00c8d33aa3..5bff5511ef6 100644 --- a/sdk/python/feast/repo_operations.py +++ b/sdk/python/feast/repo_operations.py @@ -12,7 +12,6 @@ from click.exceptions import BadParameter from feast import Entity, FeatureTable -from feast.errors import DataSourceNotFoundException from feast.feature_view import FeatureView from feast.inference import ( infer_entity_value_type_from_feature_views, @@ -164,6 +163,7 @@ def apply_total(repo_config: RepoConfig, repo_path: Path): update_data_sources_with_inferred_event_timestamp_col(data_sources) for view in repo.feature_views: view.infer_features_from_input_source() + data_source.validate() tables_to_delete = [] for registry_table in registry.list_feature_tables(project=project): diff --git a/sdk/python/tests/test_cli_gcp.py b/sdk/python/tests/test_cli_gcp.py index 946cf863b2a..ac005e3c36a 100644 --- a/sdk/python/tests/test_cli_gcp.py +++ b/sdk/python/tests/test_cli_gcp.py @@ -80,9 +80,11 @@ def test_missing_bq_source_fail() -> None: repo_example = repo_path / "example.py" repo_example.write_text( - (Path(__file__).parent / "example_feature_repo_with_missing_bq_source.py").read_text() + ( + Path(__file__).parent / "example_feature_repo_with_missing_bq_source.py" + ).read_text() ) returncode, output = runner.run_with_output(["apply"], cwd=repo_path) assert returncode == 1 - assert "DataSourceNotFoundException" in output + assert b"DataSourceNotFoundException" in output From 458beab6945ec140b08bd5bfd668329d03869df9 Mon Sep 17 00:00:00 2001 From: Cody Lin Date: Fri, 18 Jun 2021 20:07:14 -0700 Subject: [PATCH 18/20] fixed tests Signed-off-by: Cody Lin --- sdk/python/feast/data_source.py | 34 ----------------------------- sdk/python/feast/repo_operations.py | 2 +- 2 files changed, 1 insertion(+), 35 deletions(-) diff --git a/sdk/python/feast/data_source.py b/sdk/python/feast/data_source.py index ccedca37025..c25b64c82f4 100644 --- a/sdk/python/feast/data_source.py +++ b/sdk/python/feast/data_source.py @@ -526,40 +526,6 @@ def validate(self): """ raise NotImplementedError - def _infer_event_timestamp_column(self, ts_column_type_regex_pattern): - ERROR_MSG_PREFIX = "Unable to infer DataSource event_timestamp_column" - USER_GUIDANCE = "Please specify event_timestamp_column explicitly." - - if isinstance(self, FileSource) or isinstance(self, BigQuerySource): - event_timestamp_column, matched_flag = None, False - for col_name, col_datatype in self.get_table_column_names_and_types(): - if re.match(ts_column_type_regex_pattern, col_datatype): - if matched_flag: - raise TypeError( - f""" - {ERROR_MSG_PREFIX} due to multiple possible columns satisfying - the criteria. {USER_GUIDANCE} - """ - ) - matched_flag = True - event_timestamp_column = col_name - if matched_flag: - return event_timestamp_column - else: - raise TypeError( - f""" - {ERROR_MSG_PREFIX} due to an absence of columns that satisfy the criteria. - {USER_GUIDANCE} - """ - ) - else: - raise TypeError( - f""" - {ERROR_MSG_PREFIX} because this DataSource currently does not support this inference. - {USER_GUIDANCE} - """ - ) - class FileSource(DataSource): def __init__( diff --git a/sdk/python/feast/repo_operations.py b/sdk/python/feast/repo_operations.py index 5bff5511ef6..3ed219138b5 100644 --- a/sdk/python/feast/repo_operations.py +++ b/sdk/python/feast/repo_operations.py @@ -159,11 +159,11 @@ def apply_total(repo_config: RepoConfig, repo_path: Path): assert_offline_store_supports_data_source( repo_config.offline_store, data_source ) + data_source.validate() update_data_sources_with_inferred_event_timestamp_col(data_sources) for view in repo.feature_views: view.infer_features_from_input_source() - data_source.validate() tables_to_delete = [] for registry_table in registry.list_feature_tables(project=project): From 5abc428d17b16d70d01154e76c96cb616e4b6cf0 Mon Sep 17 00:00:00 2001 From: Cody Lin Date: Sat, 19 Jun 2021 21:14:26 -0700 Subject: [PATCH 19/20] Set pytest.integration for tests that access BQ Signed-off-by: Cody Lin --- sdk/python/tests/test_cli_local.py | 2 ++ sdk/python/tests/test_online_retrieval.py | 2 ++ sdk/python/tests/test_partial_apply.py | 1 + 3 files changed, 5 insertions(+) diff --git a/sdk/python/tests/test_cli_local.py b/sdk/python/tests/test_cli_local.py index 5b1a988a99b..2000c6d4aff 100644 --- a/sdk/python/tests/test_cli_local.py +++ b/sdk/python/tests/test_cli_local.py @@ -10,6 +10,7 @@ from tests.online_read_write_test import basic_rw_test +@pytest.mark.integration def test_workflow() -> None: """ Test running apply on a sample repo, and make sure the infra gets created. @@ -78,6 +79,7 @@ def test_workflow() -> None: assertpy.assert_that(result.returncode).is_equal_to(0) +@pytest.mark.integration def test_non_local_feature_repo() -> None: """ Test running apply on a sample repo, and make sure the infra gets created. diff --git a/sdk/python/tests/test_online_retrieval.py b/sdk/python/tests/test_online_retrieval.py index 3f5df6b3e0b..d9939871e90 100644 --- a/sdk/python/tests/test_online_retrieval.py +++ b/sdk/python/tests/test_online_retrieval.py @@ -14,6 +14,7 @@ from tests.cli_utils import CliRunner, get_example_repo +@pytest.mark.integration def test_online() -> None: """ Test reading from the online store in local mode. @@ -238,6 +239,7 @@ def test_online() -> None: os.rename(store.config.registry + "_fake", store.config.registry) +@pytest.mark.integration def test_online_to_df(): """ Test dataframe conversion. Make sure the response columns and rows are diff --git a/sdk/python/tests/test_partial_apply.py b/sdk/python/tests/test_partial_apply.py index c8c9de76b12..23ea956dcad 100644 --- a/sdk/python/tests/test_partial_apply.py +++ b/sdk/python/tests/test_partial_apply.py @@ -5,6 +5,7 @@ from tests.online_read_write_test import basic_rw_test +@pytest.mark.integration def test_partial() -> None: """ Add another table to existing repo using partial apply API. Make sure both the table From 4ec510daa23856cbcc38cdcbbbf5710ce94b9045 Mon Sep 17 00:00:00 2001 From: Cody Lin Date: Sat, 19 Jun 2021 21:18:21 -0700 Subject: [PATCH 20/20] Import pytest in failed test files Signed-off-by: Cody Lin --- sdk/python/tests/test_cli_local.py | 1 + sdk/python/tests/test_partial_apply.py | 1 + 2 files changed, 2 insertions(+) diff --git a/sdk/python/tests/test_cli_local.py b/sdk/python/tests/test_cli_local.py index 2000c6d4aff..288a2462452 100644 --- a/sdk/python/tests/test_cli_local.py +++ b/sdk/python/tests/test_cli_local.py @@ -4,6 +4,7 @@ from textwrap import dedent import assertpy +import pytest from feast.feature_store import FeatureStore from tests.cli_utils import CliRunner diff --git a/sdk/python/tests/test_partial_apply.py b/sdk/python/tests/test_partial_apply.py index 23ea956dcad..062d9664186 100644 --- a/sdk/python/tests/test_partial_apply.py +++ b/sdk/python/tests/test_partial_apply.py @@ -1,3 +1,4 @@ +import pytest from google.protobuf.duration_pb2 import Duration from feast import BigQuerySource, Feature, FeatureView, ValueType