Skip to content

Provide descriptive error on invalid table reference #1627

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
feast-ci-bot merged 20 commits into from
Jun 21, 2021
Merged

Provide descriptive error on invalid table reference #1627

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
1a91a33
Initial commit to catch nonexistent table
Jun 7, 2021
da7a712
simplify nonexistent BQ table test
Jun 8, 2021
f6a2f98
clean up table_exists exception
Jun 8, 2021
41e69a7
remove unneeded variable
Jun 8, 2021
af937c4
function name change to _assert_table_exists
Jun 8, 2021
215de4b
Initial commit to catch nonexistent table
Jun 7, 2021
caac9d0
simplify nonexistent BQ table test
Jun 8, 2021
2d2a484
clean up table_exists exception
Jun 8, 2021
c548ab6
function name change to _assert_table_exists
Jun 8, 2021
d5eb8af
fix lint errors and rebase
Jun 9, 2021
21a7631
Fix get_table(None) error
Jun 9, 2021
50831ef
custom exception for both missing file and BQ source
Jun 9, 2021
4e7a803
revert FileSource checks
Jun 10, 2021
2d99b05
Use DataSourceNotFoundException instead of subclassing
Jun 11, 2021
c513660
Moved assert_table_exists out of the BQ constructor to apply_total
Jun 14, 2021
617dde1
rename test and test asset
Jun 16, 2021
6b3975a
move validate logic back to data_source
Jun 17, 2021
458beab
fixed tests
Jun 19, 2021
5abc428
Set pytest.integration for tests that access BQ
Jun 20, 2021
4ec510d
Import pytest in failed test files
Jun 20, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
custom exception for both missing file and BQ source 
Signed-off-by: Cody Lin <codyl@twitter.com>
  • Loading branch information
Cody Lin committed Jun 19, 2021
commit 50831ef692421c7bba40efca2bc8ec21a8d4c8d0
14 changes: 9 additions & 5 deletions sdk/python/feast/data_source.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@

from feast import type_map
from feast.data_format import FileFormat, StreamFormat
from feast.errors import BigQuerySourceNotFoundException, FileSourceNotFoundException
from feast.protos.feast.core.DataSource_pb2 import DataSource as DataSourceProto
from feast.value_type import ValueType

Expand Down Expand Up @@ -561,6 +562,7 @@ def __init__(
file_url = path

self._file_options = FileOptions(file_format=file_format, file_url=file_url)
self._assert_table_exists()

super().__init__(
event_timestamp_column,
Expand Down Expand Up @@ -623,6 +625,12 @@ def get_table_column_names_and_types(self) -> Iterable[Tuple[str, str]]:
schema = ParquetFile(self.path).schema_arrow
return zip(schema.names, map(str, schema.types))

def _assert_table_exists(self):
try:
ParquetFile(self.path).schema_arrow
except (FileNotFoundError):
raise FileSourceNotFoundException(self.path)


class BigQuerySource(DataSource):
def __init__(
Expand Down Expand Up @@ -734,11 +742,7 @@ def _assert_table_exists(self):
try:
client.get_table(self.table_ref)
except NotFound:
raise NameError(
f"""
Unable to find table {self.table_ref} in BigQuery. Please check that table exists.
"""
)
raise BigQuerySourceNotFoundException(self.table_ref)


class KafkaSource(DataSource):
Expand Down
16 changes: 16 additions & 0 deletions sdk/python/feast/errors.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,22 @@
from colorama import Fore, Style


class DataSourceNotFoundException(Exception):
pass


class BigQuerySourceNotFoundException(DataSourceNotFoundException):
Copy link
Copy Markdown
Member

@woop woop Jun 10, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not sure if it's a good idea to have implementation specific errors. We are moving to having all implementations be self-contained. Any downside to using DataSourceNotFoundException instead of subclassing?

Copy link
Copy Markdown
Contributor Author

@codyjlin codyjlin Jun 11, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah thanks for letting me know - I was following the example of the FeastObjectNotFoundException subclassing pattern here, but since the DataSourceNotFoundExceptions are so similar (and we're not checking FileSource yet), I'll just change it to use DataSourceNotFoundException instead.

def __init__(self, table_ref):
super().__init__(
f"Unable to find table '{table_ref}' in BigQuery. Please check that table exists."
)


class FileSourceNotFoundException(DataSourceNotFoundException):
def __init__(self, path):
super().__init__(f"Unable to find table at '{path}'")


class FeastObjectNotFoundException(Exception):
pass

Expand Down
19 changes: 16 additions & 3 deletions sdk/python/tests/test_data_source.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,20 @@
import pytest
from utils.data_source_utils import simple_bq_source_using_table_ref_arg
from utils.data_source_utils import (
prep_file_source,
simple_bq_source_using_table_ref_arg,
)

from feast.data_source import BigQuerySource
from feast.data_source import BigQuerySource, FileSource
from feast.errors import BigQuerySourceNotFoundException, FileSourceNotFoundException


def test_existent_file_source(simple_dataset_1):
prep_file_source(df=simple_dataset_1)


def test_nonexistent_file_source(simple_dataset_1):
with pytest.raises(FileSourceNotFoundException):
FileSource(file_url="nonexistent_file")


@pytest.mark.integration
Expand All @@ -11,7 +24,7 @@ def test_existent_bq_source(simple_dataset_1):

@pytest.mark.integration
def test_nonexistent_bq_source():
with pytest.raises(NameError):
with pytest.raises(BigQuerySourceNotFoundException):
BigQuerySource(
table_ref="project.dataset.nonexistent_table", event_timestamp_column=""
)