Merge: [WIDENDEEP/TF1] Bump Spark from 2.3.2 to 3.1.3

nv-kkudrynski · nv-kkudrynski · commit 174b3d40bfc2 · 2022-06-30T07:21:33.000-07:00
diff --git a/TensorFlow/Recommendation/WideAndDeep/Dockerfile b/TensorFlow/Recommendation/WideAndDeep/Dockerfile
@@ -19,7 +19,7 @@ FROM ${FROM_IMAGE_NAME}
 USER root
 
 # Spark dependencies
-ENV APACHE_SPARK_VERSION 2.3.1
+ENV APACHE_SPARK_VERSION 3.1.3
 ENV HADOOP_VERSION 2.7
 
 RUN apt-get -y update && \
@@ -29,7 +29,7 @@ RUN apt-get -y update && \
 
 RUN cd /tmp && \
         wget -q http://archive.apache.org/dist/spark/spark-${APACHE_SPARK_VERSION}/spark-${APACHE_SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz && \
-        echo "DC3A97F3D99791D363E4F70A622B84D6E313BD852F6FDBC777D31EAB44CBC112CEEAA20F7BF835492FB654F48AE57E9969F93D3B0E6EC92076D1C5E1B40B4696 *spark-${APACHE_SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" | sha512sum -c - && \
+        echo "34b2c6c5698c254181a020f9e0e5d29b3edeb3b99cd7f51105760b29681461e6e1bb0490c09d8b2005b5836f22d74251961939cad010ffc0d8de656d633b976e *spark-${APACHE_SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" | sha512sum -c - && \
         tar xzf spark-${APACHE_SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz -C /usr/local --owner root --group root --no-same-owner && \
         rm spark-${APACHE_SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz
 RUN cd /usr/local && ln -s spark-${APACHE_SPARK_VERSION}-bin-hadoop${HADOOP_VERSION} spark
diff --git a/TensorFlow/Recommendation/WideAndDeep/requirements.txt b/TensorFlow/Recommendation/WideAndDeep/requirements.txt
@@ -1,3 +1,3 @@
 git+https://github.com/NVIDIA/dllogger@v1.0.0#egg=dllogger
 ipdb==0.13.9
-pyspark==2.3.2
+pyspark==3.1.3
