Bundle python dependencies for spark images (#137)

khorshuheng · web-flow · commit eee8af860da6 · 2022-04-21T16:27:00.000+08:00
Signed-off-by: Khor Shu Heng &lt;khor.heng@gojek.com&gt;

Co-authored-by: Khor Shu Heng &lt;khor.heng@gojek.com&gt;
diff --git a/infra/docker/spark/Dockerfile b/infra/docker/spark/Dockerfile
@@ -36,10 +36,10 @@ RUN echo 'spark.executor.extraJavaOptions="-Dio.netty.tryReflectionSetAccessible
 RUN echo 'spark.executor.extraJavaOptions="-Dcom.google.cloud.spark.bigquery.repackaged.io.netty.tryReflectionSetAccessible=true"' >> $SPARK_HOME/conf/spark-defaults.conf
 
 # python dependencies
-RUN pip3 install -U pip wheel
-RUN pip3 install pandas pyarrow==2.0.0 'numpy<1.20.0'
+RUN pip3 install pandas==1.3.5 great-expectations==0.13.2 pyarrow==2.0.0 Jinja2==3.0.3 datadog==0.44.0 'numpy<1.20.0'
 
 # For logging to /dev/termination-log
 RUN mkdir -p /dev
 
+
 ENTRYPOINT [ "/opt/entrypoint.sh" ]
diff --git a/infra/docker/spark/dev.Dockerfile b/infra/docker/spark/dev.Dockerfile
@@ -21,11 +21,10 @@ RUN mkdir -p /opt/spark/conf
 RUN echo 'spark.driver.extraJavaOptions="-Dio.netty.tryReflectionSetAccessible=true"' >> $SPARK_HOME/conf/spark-defaults.conf
 RUN echo 'spark.driver.extraJavaOptions="-Dcom.google.cloud.spark.bigquery.repackaged.io.netty.tryReflectionSetAccessible=true"' >> $SPARK_HOME/conf/spark-defaults.conf
 RUN echo 'spark.executor.extraJavaOptions="-Dio.netty.tryReflectionSetAccessible=true"' >> $SPARK_HOME/conf/spark-defaults.conf
-RUN echo 'spark.executor.extraJavaOptions="-Dcom.google.cloud.spark.bigquery.repackaged.io.netty.tryReflectionSetAccessible=true"' >> $SPARK_HOME/conf/spark-defaults.conf
+RUN echo 'spark.executor.extraJavaOptions="-Dcom.google.cloud.spark.bigquery.repackaged.io.netty.tryReflectionSetAccessible=true"' >> $SPARK_HOME/conf/spark-defaults.conf \
 
 # python dependencies
-RUN pip3 install -U pip wheel
-RUN pip3 install pandas pyarrow==2.0.0 'numpy<1.20.0'
+RUN pip3 install pandas==1.3.5 great-expectations==0.13.2 pyarrow==2.0.0 Jinja2==3.0.3 datadog==0.44.0 'numpy<1.20.0'
 
 # For logging to /dev/termination-log
 RUN mkdir -p /dev
diff --git a/tests/e2e/test_validation.py b/tests/e2e/test_validation.py
@@ -52,7 +52,9 @@ def test_validation_with_ge(
     expectations = ge_ds.get_expectation_suite()
 
     udf = create_validation_udf("testUDF", expectations, feature_table)
-    apply_validation(feast_client, feature_table, udf, validation_window_secs=1)
+    apply_validation(
+        feast_client, feature_table, udf, validation_window_secs=1, include_py_libs=""
+    )
 
     job = start_job(feast_spark_client, feature_table, pytestconfig)
 
@@ -123,7 +125,9 @@ def test_validation_reports_metrics(
     expectations = ge_ds.get_expectation_suite()
 
     udf = create_validation_udf("testUDF", expectations, feature_table)
-    apply_validation(feast_client, feature_table, udf, validation_window_secs=10)
+    apply_validation(
+        feast_client, feature_table, udf, validation_window_secs=10, include_py_libs=""
+    )
 
     job = start_job(feast_spark_client, feature_table, pytestconfig)
 
