feast-dev · feast-ci-bot · Dec 14, 2020 · Dec 14, 2020 · Dec 14, 2020 · Dec 14, 2020
diff --git a/.github/workflows/complete.yml b/.github/workflows/complete.yml
@@ -75,15 +75,20 @@ jobs:
   unit-test-java:
     runs-on: ubuntu-latest
     needs: lint-java
-    container: gcr.io/kf-feast/feast-ci:latest
     steps:
       - uses: actions/checkout@v2
-      - uses: actions/cache@v1
+      - name: Set up JDK 11
+        uses: actions/setup-java@v1
+        with:
+          java-version: '11'
+          java-package: jdk
+          architecture: x64
+      - uses: actions/cache@v2
         with:
           path: ~/.m2/repository
-          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
+          key: ${{ runner.os }}-ut-maven-${{ hashFiles('**/pom.xml') }}
           restore-keys: |
-            ${{ runner.os }}-maven-
+            ${{ runner.os }}-ut-maven-
       - name: Test java
         run: make test-java-with-coverage
       - uses: actions/upload-artifact@v2
@@ -118,12 +123,18 @@ jobs:
     needs: unit-test-java
     steps:
       - uses: actions/checkout@v2
-      - name: Set up JDK 1.8
+      - name: Set up JDK 11
         uses: actions/setup-java@v1
         with:
           java-version: '11'
           java-package: jdk
           architecture: x64
+      - uses: actions/cache@v2
+        with:
+          path: ~/.m2/repository
+          key: ${{ runner.os }}-it-maven-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            ${{ runner.os }}-it-maven-
       - name: Run integration tests
         run:  make test-java-integration
 

@@ -1,4 +1,4 @@
-FROM jupyter/pyspark-notebook:ae5f7e104dd5
+FROM jupyter/pyspark-notebook:399cbb986c6b
 
 USER root
 WORKDIR /feast

@@ -1,4 +1,4 @@
-FROM jupyter/pyspark-notebook:ae5f7e104dd5
+FROM jupyter/pyspark-notebook:399cbb986c6b
 
 USER root
 WORKDIR /feast

@@ -107,7 +107,7 @@ def _sync_offline_to_online_step(
                 "--class",
                 "feast.ingestion.IngestionJob",
                 "--packages",
-                "com.google.cloud.spark:spark-bigquery-with-dependencies_2.12:0.17.2",
+                "com.google.cloud.spark:spark-bigquery-with-dependencies_2.12:0.18.0",
                 jar_path,
             ]
             + args,
@@ -332,7 +332,7 @@ def _stream_ingestion_step(
             + jars_args
             + [
                 "--packages",
-                "com.google.cloud.spark:spark-bigquery-with-dependencies_2.12:0.17.2",
+                "com.google.cloud.spark:spark-bigquery-with-dependencies_2.12:0.18.0",
                 jar_path,
             ]
             + args,

@@ -222,7 +222,7 @@ class StandaloneClusterLauncher(JobLauncher):
     Submits jobs to a standalone Spark cluster in client mode.
     """
 
-    BQ_CONNECTOR_VERSION = "2.12:0.17.3"
+    BQ_CONNECTOR_VERSION = "2.12:0.18.0"
 
     def __init__(self, master_url: str, spark_home: str = None):
         """

@@ -4,7 +4,7 @@ black==19.10b0
 isort>=5
 grpcio-tools==1.31.0
 mypy-protobuf
-pyspark==2.4.2
+pyspark==3.0.1
 pandas~=1.0.0
 mock==2.0.0
 pandavro==1.5.*
@@ -19,4 +19,5 @@ pytest==6.0.0
 pytest-lazy-fixture==0.6.3
 pytest-timeout==1.4.2
 pytest-ordering==0.6.*
-pytest-mock==1.10.4
+pytest-mock==1.10.4
+PyYAML==5.3.1
@@ -38,5 +38,5 @@ flake8
 black==19.10b0
 boto3
 moto
-pyspark==2.4.2
-pyspark-stubs==2.4.0.post9
+pyspark==3.0.1
+pyspark-stubs==3.0.0.post1
@@ -36,12 +36,12 @@
     "pandas~=1.0.0",
     "pandavro==1.5.*",
     "protobuf>=3.10",
-    "PyYAML==5.1.*",
+    "PyYAML==5.3.*",
     "fastavro>=0.22.11,<0.23",
     "tabulate==0.8.*",
     "toml==0.10.*",
     "tqdm==4.*",
-    "pyarrow<0.16.0,>=0.15.1",
+    "pyarrow==2.0.0",
     "numpy",
     "google",
 ]

diff --git a/spark/ingestion/pom.xml b/spark/ingestion/pom.xml
@@ -33,7 +33,7 @@
     <properties>
         <scala.version>2.12</scala.version>
         <scala.fullVersion>${scala.version}.12</scala.fullVersion>
-        <spark.version>2.4.7</spark.version>
+        <spark.version>3.0.1</spark.version>
         <scala-maven-plugin.version>4.4.0</scala-maven-plugin.version>
         <maven-assembly-plugin.version>3.3.0</maven-assembly-plugin.version>
         <protobuf.version>3.12.2</protobuf.version>
@@ -98,6 +98,13 @@
             <scope>provided</scope>
         </dependency>
 
+        <dependency>
+            <groupId>org.apache.spark</groupId>
+            <artifactId>spark-catalyst_${scala.version}</artifactId>
+            <version>${spark.version}</version>
+            <scope>provided</scope>
+        </dependency>
+
         <dependency>
             <groupId>org.codehaus.janino</groupId>
             <artifactId>janino</artifactId>
@@ -131,7 +138,7 @@
         <dependency>
             <groupId>com.google.cloud.spark</groupId>
             <artifactId>spark-bigquery_${scala.version}</artifactId>
-            <version>0.17.2</version>
+            <version>0.18.0</version>
             <scope>provided</scope>
         </dependency>
 
@@ -147,18 +154,18 @@
             <version>2.5.0</version>
         </dependency>
 
-        <dependency>
-            <groupId>org.apache.arrow</groupId>
-            <artifactId>arrow-vector</artifactId>
-            <version>0.16.0</version>
-        </dependency>
-
         <dependency>
             <groupId>io.netty</groupId>
             <artifactId>netty-all</artifactId>
             <version>4.1.52.Final</version>
         </dependency>
 
+        <dependency>
+            <groupId>commons-io</groupId>
+            <artifactId>commons-io</artifactId>
+            <version>2.5</version>
+        </dependency>
+
         <dependency>
             <groupId>org.json4s</groupId>
             <artifactId>json4s-ext_${scala.version}</artifactId>

diff --git a/spark/ingestion/src/main/scala/feast/ingestion/BasePipeline.scala b/spark/ingestion/src/main/scala/feast/ingestion/BasePipeline.scala
@@ -56,6 +56,8 @@ trait BasePipeline {
           .set("spark.metrics.conf.*.sink.statsd.period", "30")
           .set("spark.metrics.conf.*.sink.statsd.unit", "seconds")
           .set("spark.metrics.namespace", jobConfig.mode.toString.toLowerCase)
+          // until proto parser udf will be fixed, we have to use this
+          .set("spark.sql.legacy.allowUntypedScalaUDF", "true")
       case None => ()
     }
 

diff --git a/spark/ingestion/src/main/scala/feast/ingestion/StreamingPipeline.scala b/spark/ingestion/src/main/scala/feast/ingestion/StreamingPipeline.scala
@@ -119,6 +119,8 @@ object StreamingPipeline extends BasePipeline with Serializable {
 
     val parser: Array[Byte] => Row = ProtoReflection.createMessageParser(protoRegistry, className)
 
+    // ToDo: create correctly typed parser
+    // spark deprecated returnType argument, instead it will infer it from udf function signature
     udf(parser, ProtoReflection.inferSchema(protoRegistry.getProtoDescriptor(className)))
   }
 }
diff --git a/spark/ingestion/src/test/scala/feast/ingestion/SparkSpec.scala b/spark/ingestion/src/test/scala/feast/ingestion/SparkSpec.scala
@@ -39,6 +39,7 @@ class SparkSpec extends UnitSpec with BeforeAndAfter {
       )
       .set("spark.metrics.conf.*.sink.statsd.host", "localhost")
       .set("spark.metrics.conf.*.sink.statsd.port", "8125")
+      .set("spark.sql.legacy.allowUntypedScalaUDF", "true")
 
     sparkSession = SparkSession
       .builder()