fix: Add __eq__, __hash__ to SparkSource for correct comparison (#4028)

ElliotNguyen68 · web-flow · commit e703b40582e6 · 2024-03-25T19:33:58.000-07:00
* feat: Enable Arrow-based columnar data transfers

Signed-off-by: tanlocnguyen &lt;tanlocnguyen296@gmail.com&gt;

* fix: Add __eq__, __hash__ to SparkSource for comparision

Signed-off-by: tanlocnguyen &lt;tanlocnguyen296@gmail.com&gt;

* chore: simplify the logic

Signed-off-by: tanlocnguyen &lt;tanlocnguyen296@gmail.com&gt;

---------

Signed-off-by: tanlocnguyen &lt;tanlocnguyen296@gmail.com&gt;
Co-authored-by: tanlocnguyen &lt;tanlocnguyen296@gmail.com&gt;
diff --git a/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark_source.py b/sdk/python/feast/infra/offline_stores/contrib/spark_offline_store/spark_source.py
@@ -185,6 +185,19 @@ def get_table_query_string(self) -> str:
 
         return f"`{tmp_table_name}`"
 
+    def __eq__(self, other):
+        base_eq = super().__eq__(other)
+        if not base_eq:
+            return False
+        return (
+            self.table == other.table
+            and self.query == other.query
+            and self.path == other.path
+        )
+
+    def __hash__(self):
+        return super().__hash__()
+
 
 class SparkOptions:
     allowed_formats = [format.value for format in SparkSourceFormat]
