Skip to content
Merged
Prev Previous commit
Next Next commit
add tests
Signed-off-by: HaoXuAI <sduxuhao@gmail.com>
  • Loading branch information
HaoXuAI committed Oct 5, 2025
commit 7a34eafdd936e89c750def26c6af0e8b1d6fe900
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,7 @@ def from_proto(data_source: DataSourceProto) -> Any:
query=spark_options.query,
path=spark_options.path,
file_format=spark_options.file_format,
table_format=spark_options.table_format,
date_partition_column_format=spark_options.date_partition_column_format,
date_partition_column=data_source.date_partition_column,
timestamp_field=data_source.timestamp_field,
Expand Down Expand Up @@ -369,15 +370,20 @@ def from_proto(cls, spark_options_proto: DataSourceProto.SparkOptions):
Returns:
Returns a SparkOptions object based on the spark_options protobuf
"""
# Parse table_format if present
table_format = None
if spark_options_proto.table_format:
table_format = table_format_from_dict(
json.loads(spark_options_proto.table_format)
)

spark_options = cls(
table=spark_options_proto.table,
query=spark_options_proto.query,
path=spark_options_proto.path,
file_format=spark_options_proto.file_format,
date_partition_column_format=spark_options_proto.date_partition_column_format,
table_format=table_format_from_dict(
json.loads(spark_options_proto.table_format)
),
table_format=table_format,
)

return spark_options
Expand Down
Loading
Loading