feast-dev
diff --git a/‎.github/workflows/complete.yml‎
Lines changed: 2 additions & 27 deletions b/‎.github/workflows/complete.yml‎
Lines changed: 2 additions & 27 deletions
diff --git a/‎spark/ingestion/src/main/scala/feast/ingestion/stores/redis/RedisSinkRelation.scala‎
Lines changed: 0 additions & 1 deletion b/‎spark/ingestion/src/main/scala/feast/ingestion/stores/redis/RedisSinkRelation.scala‎
Lines changed: 0 additions & 1 deletion
diff --git a/‎spark/ingestion/src/main/scala/org/apache/spark/api/python/DynamicPythonFunction.scala‎
Lines changed: 6 additions & 2 deletions b/‎spark/ingestion/src/main/scala/org/apache/spark/api/python/DynamicPythonFunction.scala‎
Lines changed: 6 additions & 2 deletions
@@ -144,9 +144,9 @@ jobs:
         uses: actions/upload-artifact@v2
         if: failure()
         with:
-          name: ingestion-it-report
+          name: it-report
           path: spark/ingestion/target/test-reports/TestSuite.txt
-          retention-days: 1
+          retention-days: 5
 
   tests-docker-compose:
     needs:
@@ -193,28 +193,3 @@ jobs:
           name: ingestion-jar
           path: spark/ingestion/target/feast-ingestion-spark-develop.jar
           retention-days: 1
-
-  publish-ingestion-pylibs:
-    strategy:
-      matrix:
-        os: [ ubuntu-latest, macos-latest ]
-        python-version: [ 3.6, 3.7, 3.8 ]
-    runs-on: ${{ matrix.os }}
-    steps:
-      - uses: actions/checkout@v2
-      - name: Set up Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python-version }}
-      - name: Create libs archive
-        env:
-          PY_VERSION: ${{ matrix.python-version }}
-        run: |
-          export PLATFORM=$(python -c 'import platform; print(platform.system().lower())')
-          mkdir pylibs/
-          ./infra/scripts/build-ingestion-py-dependencies.sh "py${PY_VERSION}-$PLATFORM" $PWD/pylibs/
-      - uses: actions/upload-artifact@v2
-        with:
-          name: pylibs
-          path: pylibs/
-          retention-days: 1
@@ -83,7 +83,6 @@ class RedisSinkRelation(override val sqlContext: SQLContext, config: SparkRedisC
 
         groupKeysByNode(redisConfig.hosts, rowsWithKey.keysIterator).foreach { case (node, keys) =>
           val conn = node.connect()
-
           // retrieve latest stored values
           val storedValues = mapWithPipeline(conn, keys) { (pipeline, key) =>
             persistence.get(pipeline, key.toByteArray)
 
@@ -22,14 +22,18 @@ import java.util.{ArrayList => JArrayList, HashMap => JHashMap}
 import org.apache.commons.io.IOUtils
 import org.apache.spark.SparkEnv
 import org.apache.spark.broadcast.Broadcast
-import org.apache.spark.internal.config.PYSPARK_DRIVER_PYTHON
+import org.apache.spark.internal.config.{PYSPARK_DRIVER_PYTHON, PYSPARK_PYTHON}
 
 import collection.JavaConverters._
 
 object DynamicPythonFunction {
   private val conf = SparkEnv.get.conf
 
-  val pythonExec = conf.get(PYSPARK_DRIVER_PYTHON).getOrElse("python3")
+  val pythonExec = conf.get(PYSPARK_DRIVER_PYTHON)
+    .orElse(conf.get(PYSPARK_PYTHON))
+    .orElse(sys.env.get("PYSPARK_DRIVER_PYTHON"))
+    .orElse(sys.env.get("PYSPARK_PYTHON"))
+    .getOrElse("python3")
 
   private def runCommand(cmd: List[String]): String = {
     val pb = new ProcessBuilder(cmd.asJava)