Skip to content

fix: Remove date partition column field from datasources that don't s… #2478

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
feast-ci-bot merged 11 commits into from
Apr 4, 2022
Merged

fix: Remove date partition column field from datasources that don't s… #2478

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
33245bc
fix: Remove date partition column field from datasources that don't s…
achals Apr 1, 2022
319b073
remove for file sources as well
achals Apr 1, 2022
d3aef10
remove other references
achals Apr 1, 2022
462c210
remove other references
achals Apr 1, 2022
d655ada
reference to removal
achals Apr 1, 2022
ed96a2b
reference to removal
achals Apr 1, 2022
06f81ed
reorder
achals Apr 1, 2022
0b7e26e
remove more:
achals Apr 1, 2022
300f76b
docs
achals Apr 1, 2022
2da035e
comment
achals Apr 1, 2022
c2dbd7b
cr
achals Apr 1, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
fix: Remove date partition column field from datasources that don't s… 
…upport them

Signed-off-by: Achal Shah <achals@gmail.com>
  • Loading branch information
@achals
achals committed Apr 4, 2022
commit 33245bcd8856713ccf597aee5a59930cd026a858
18 changes: 12 additions & 6 deletions sdk/python/feast/infra/offline_stores/bigquery_source.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@ def __init__(
table: Optional[str] = None,
table_ref: Optional[str] = None,
created_timestamp_column: Optional[str] = "",
date_partition_column: Optional[str] = None,
Comment thread
felixwang9817 marked this conversation as resolved.
Outdated
field_mapping: Optional[Dict[str, str]] = None,
date_partition_column: Optional[str] = "",
query: Optional[str] = None,
name: Optional[str] = None,
description: Optional[str] = "",
Expand All @@ -37,7 +37,6 @@ def __init__(
created_timestamp_column (optional): Timestamp column when row was created, used for deduplicating rows.
field_mapping: A dictionary mapping of column names in this data source to feature names in a feature table
or view. Only used for feature columns, not entities or timestamp columns.
date_partition_column (optional): Timestamp column used for partitioning.
query (optional): SQL query to execute to generate data for this data source.
name (optional): Name for the source. Defaults to the table_ref if not specified.
description (optional): A human-readable description.
Expand All @@ -61,6 +60,14 @@ def __init__(
table = table_ref
self.bigquery_options = BigQueryOptions(table_ref=table, query=query)

if date_partition_column:
warnings.warn(
Comment thread
felixwang9817 marked this conversation as resolved.
Outdated
(
"The argument 'date_partition_column' is not supported for BigQuery sources."
),
DeprecationWarning,
)

# If no name, use the table_ref as the default name
_name = name
if not _name:
Expand All @@ -78,10 +85,9 @@ def __init__(

super().__init__(
_name if _name else "",
event_timestamp_column,
created_timestamp_column,
field_mapping,
date_partition_column,
Comment thread
felixwang9817 marked this conversation as resolved.
Outdated
event_timestamp_column=event_timestamp_column,
created_timestamp_column=created_timestamp_column,
field_mapping=field_mapping,
description=description,
tags=tags,
owner=owner,
Expand Down
20 changes: 13 additions & 7 deletions sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark_source.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,12 +51,20 @@ def __init__(
_name = table
else:
raise DataSourceNoNameException()

if date_partition_column:
warnings.warn(
(
"The argument 'date_partition_column' is not supported for Spark sources."
),
DeprecationWarning,
)

super().__init__(
_name,
event_timestamp_column,
created_timestamp_column,
field_mapping,
date_partition_column,
name=_name,
event_timestamp_column=event_timestamp_column,
created_timestamp_column=created_timestamp_column,
field_mapping=field_mapping,
description=description,
tags=tags,
owner=owner,
Expand Down Expand Up @@ -130,7 +138,6 @@ def from_proto(data_source: DataSourceProto) -> Any:
file_format=spark_options.file_format,
event_timestamp_column=data_source.event_timestamp_column,
created_timestamp_column=data_source.created_timestamp_column,
date_partition_column=data_source.date_partition_column,
description=data_source.description,
tags=dict(data_source.tags),
owner=data_source.owner,
Expand All @@ -149,7 +156,6 @@ def to_proto(self) -> DataSourceProto:

data_source_proto.event_timestamp_column = self.event_timestamp_column
data_source_proto.created_timestamp_column = self.created_timestamp_column
data_source_proto.date_partition_column = self.date_partition_column

return data_source_proto

Expand Down
16 changes: 11 additions & 5 deletions sdk/python/feast/infra/offline_stores/redshift_source.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@ def __init__(
table: Optional[str] = None,
schema: Optional[str] = None,
created_timestamp_column: Optional[str] = "",
date_partition_column: Optional[str] = None,
field_mapping: Optional[Dict[str, str]] = None,
date_partition_column: Optional[str] = "",
query: Optional[str] = None,
name: Optional[str] = None,
description: Optional[str] = "",
Expand Down Expand Up @@ -68,13 +68,19 @@ def __init__(
),
DeprecationWarning,
)
if date_partition_column:
warnings.warn(
(
"The argument 'date_partition_column' is not supported for Redshift sources."
),
DeprecationWarning,
)

super().__init__(
_name if _name else "",
event_timestamp_column,
created_timestamp_column,
field_mapping,
date_partition_column,
event_timestamp_column=event_timestamp_column,
created_timestamp_column=created_timestamp_column,
field_mapping=field_mapping,
description=description,
tags=tags,
owner=owner,
Expand Down
19 changes: 12 additions & 7 deletions sdk/python/feast/infra/offline_stores/snowflake_source.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,9 @@ def __init__(
table: Optional[str] = None,
query: Optional[str] = None,
event_timestamp_column: Optional[str] = "",
date_partition_column: Optional[str] = None,
created_timestamp_column: Optional[str] = "",
field_mapping: Optional[Dict[str, str]] = None,
date_partition_column: Optional[str] = "",
name: Optional[str] = None,
description: Optional[str] = "",
tags: Optional[Dict[str, str]] = None,
Expand Down Expand Up @@ -72,12 +72,19 @@ def __init__(
DeprecationWarning,
)

if date_partition_column:
warnings.warn(
(
"The argument 'date_partition_column' is not supported for Snowflake sources."
),
DeprecationWarning,
)

super().__init__(
_name if _name else "",
event_timestamp_column,
created_timestamp_column,
field_mapping,
date_partition_column,
event_timestamp_column=event_timestamp_column,
created_timestamp_column=created_timestamp_column,
field_mapping=field_mapping,
description=description,
tags=tags,
owner=owner,
Expand All @@ -101,7 +108,6 @@ def from_proto(data_source: DataSourceProto):
table=data_source.snowflake_options.table,
event_timestamp_column=data_source.event_timestamp_column,
created_timestamp_column=data_source.created_timestamp_column,
date_partition_column=data_source.date_partition_column,
query=data_source.snowflake_options.query,
description=data_source.description,
tags=dict(data_source.tags),
Expand Down Expand Up @@ -170,7 +176,6 @@ def to_proto(self) -> DataSourceProto:

data_source_proto.event_timestamp_column = self.event_timestamp_column
data_source_proto.created_timestamp_column = self.created_timestamp_column
data_source_proto.date_partition_column = self.date_partition_column

return data_source_proto

Expand Down