feat: Add decimal datatype to spark mapping to feast data type

ElliotNguyen68 · ElliotNguyen68 · commit d4637c5c8fe3 · 2024-04-08T21:19:41.000+07:00
Signed-off-by: tanlocnguyen &lt;tanlocnguyen296@gmail.com&gt;
diff --git a/sdk/python/feast/type_map.py b/sdk/python/feast/type_map.py
@@ -752,7 +752,7 @@ def _non_empty_value(value: Any) -> bool:
 
 def spark_to_feast_value_type(spark_type_as_str: str) -> ValueType:
     # TODO not all spark types are convertible
-    # Current non-convertible types: interval, map, struct, structfield, decimal, binary
+    # Current non-convertible types: interval, map, struct, structfield, binary
     type_map: Dict[str, ValueType] = {
         "null": ValueType.UNKNOWN,
         "byte": ValueType.BYTES,
@@ -762,6 +762,7 @@ def spark_to_feast_value_type(spark_type_as_str: str) -> ValueType:
         "bigint": ValueType.INT64,
         "long": ValueType.INT64,
         "double": ValueType.DOUBLE,
+        "decimal": ValueType.DOUBLE,
         "float": ValueType.FLOAT,
         "boolean": ValueType.BOOL,
         "timestamp": ValueType.UNIX_TIMESTAMP,
@@ -774,6 +775,9 @@ def spark_to_feast_value_type(spark_type_as_str: str) -> ValueType:
         "array<boolean>": ValueType.BOOL_LIST,
         "array<timestamp>": ValueType.UNIX_TIMESTAMP_LIST,
     }
+    if spark_type_as_str.startswith('decimal'):
+        spark_type_as_str = "decimal"
+
     # TODO: Find better way of doing this.
     if not isinstance(spark_type_as_str, str) or spark_type_as_str not in type_map:
         return ValueType.NULL
