Explicitly close spark session after batch ingestion is done

pyalex · pyalex · commit b8f0313d8ee9 · 2021-04-27T19:18:14.000+08:00
Signed-off-by: Oleksii Moskalenko &lt;moskalenko.alexey@gmail.com&gt;
diff --git a/spark/ingestion/src/main/scala/feast/ingestion/IngestionJob.scala b/spark/ingestion/src/main/scala/feast/ingestion/IngestionJob.scala
@@ -114,6 +114,7 @@ object IngestionJob {
           case Modes.Offline =>
             val sparkSession = BasePipeline.createSparkSession(config)
             BatchPipeline.createPipeline(sparkSession, config)
+            sparkSession.close()
           case Modes.Online =>
             val sparkSession = BasePipeline.createSparkSession(config)
             StreamingPipeline.createPipeline(sparkSession, config).get.awaitTermination
