[DAT-1492] Make bigtable keys deterministic (#145)

khorshuheng · web-flow · commit fef7c574c428 · 2022-05-18T09:13:56.000+08:00
Signed-off-by: Khor Shu Heng &lt;khor.heng@gojek.com&gt;

Co-authored-by: Khor Shu Heng &lt;khor.heng@gojek.com&gt;
diff --git a/spark/ingestion/src/main/scala/feast/ingestion/stores/bigtable/BigTableSinkRelation.scala b/spark/ingestion/src/main/scala/feast/ingestion/stores/bigtable/BigTableSinkRelation.scala
@@ -109,13 +109,13 @@ class BigTableSinkRelation(
 
     val featureColumns = featureFields.map(f => data(f.name))
 
-    val entityColumns   = config.entityColumns.map(c => data(c).cast(StringType))
-    val schema          = serializer.convertSchema(StructType(featureFields))
-    val schemaReference = serializer.schemaReference(schema)
+    val sortedEntityColumns = config.entityColumns.sorted.map(c => data(c).cast(StringType))
+    val schema              = serializer.convertSchema(StructType(featureFields))
+    val schemaReference     = serializer.schemaReference(schema)
 
     data
       .select(
-        joinEntityKey(struct(entityColumns: _*)).alias("key"),
+        joinEntityKey(struct(sortedEntityColumns: _*)).alias("key"),
         serializer.serializeData(schema)(struct(featureColumns: _*)).alias("value"),
         col(config.timestampColumn).alias("ts")
       )
