Add comments

Signed-off-by: Kevin Zhang <kzhang@tecton.ai>
feast-dev · adchia · Mar 4, 2022 · Feb 25, 2022 · Feb 26, 2022 · Feb 26, 2022
commit d790a1cee272c1976d56ee8ec7d44d0ec00d48c8
@@ -592,11 +592,13 @@ def _non_empty_value(value: Any) -> bool:
 
 def spark_to_feast_value_type(spark_type_as_str: str) -> ValueType:
     # TODO not all spark types are convertible
+    # Current non-convertible types: interval, map, struct, structfield, decimal, binary
     type_map: Dict[str, ValueType] = {
         "null": ValueType.UNKNOWN,
         "byte": ValueType.BYTES,
         "string": ValueType.STRING,
         "int": ValueType.INT32,
+        "short": ValueType.INT32,
         "bigint": ValueType.INT64,
         "long": ValueType.INT64,
         "double": ValueType.DOUBLE,
@@ -612,7 +614,7 @@ def spark_to_feast_value_type(spark_type_as_str: str) -> ValueType:
         "array<boolean>": ValueType.BOOL_LIST,
         "array<timestamp>": ValueType.UNIX_TIMESTAMP_LIST,
     }
-    # TODO: this is just incorrect fix
+    # TODO: Find better way of doing this.
     if type(spark_type_as_str) != str or spark_type_as_str not in type_map:
         return ValueType.NULL
     return type_map[spark_type_as_str.lower()]

@@ -78,8 +78,6 @@ def create_data_source(
                 col = timestamp_mapping[event_timestamp_column]
                 df[col] = pd.to_datetime(df[col], utc=True)
 
-        # https://stackoverflow.com/questions/51871200/analysisexception-it-is-not-allowed-to-add-database-prefix
-        # destination_name = self.get_prefixed_table_name(destination_name)
         if not self.spark_session:
             self.spark_session = (
                 SparkSession.builder.config(
@@ -98,14 +96,10 @@ def create_data_source(
             event_timestamp_column=event_timestamp_column,
             created_timestamp_column=created_timestamp_column,
             date_partition_column="",
-            # feature_view => datasource accompanied
             # maps certain column names to other names
             field_mapping=field_mapping or {"ts_1": "ts"},
         )
 
     def create_saved_dataset_destination(self) -> SavedDatasetSparkStorage:
         table = f"persisted_{str(uuid.uuid4()).replace('-', '_')}"
         return SavedDatasetSparkStorage(table_ref=table, query="")
-
-    def get_prefixed_table_name(self, suffix: str) -> str:
-        return f"{self.project_name}.{suffix}"