Add clarifying comments to the Dockerfile. (GoogleCloudPlatform#11277)

tvalentyn · web-flow · commit de7a10c48b6a · 2024-03-21T12:13:08.000-07:00
* Add clarifying comments to the dockerfile.

* Opt out 3.12.
diff --git a/dataflow/flex-templates/streaming_beam/Dockerfile b/dataflow/flex-templates/streaming_beam/Dockerfile
@@ -14,22 +14,29 @@
 
 FROM gcr.io/dataflow-templates-base/python3-template-launcher-base
 
+# Configure the Template to launch the pipeline with a --requirements_file option.
+# See: https://beam.apache.org/documentation/sdks/python-pipeline-dependencies/#pypi-dependencies
 ENV FLEX_TEMPLATE_PYTHON_REQUIREMENTS_FILE="/template/requirements.txt"
 ENV FLEX_TEMPLATE_PYTHON_PY_FILE="/template/streaming_beam.py"
 
 COPY . /template
 
-# We could get rid of installing libffi-dev and git, or we could leave them.
 RUN apt-get update \
+    # Install any apt packages if required by your template pipeline.
     && apt-get install -y libffi-dev git \
     && rm -rf /var/lib/apt/lists/* \
     # Upgrade pip and install the requirements.
     && pip install --no-cache-dir --upgrade pip \
+    # Install dependencies from requirements file in the launch environment.
     && pip install --no-cache-dir -r $FLEX_TEMPLATE_PYTHON_REQUIREMENTS_FILE \
-    # Download the requirements to speed up launching the Dataflow job.
+    # When FLEX_TEMPLATE_PYTHON_REQUIREMENTS_FILE  option is used,
+    # then during Template launch Beam downloads dependencies
+    # into a local requirements cache folder and stages the cache to workers.
+    # To speed up Flex Template launch, pre-download the requirements cache
+    # when creating the Template.
     && pip download --no-cache-dir --dest /tmp/dataflow-requirements-cache -r $FLEX_TEMPLATE_PYTHON_REQUIREMENTS_FILE
 
-# Since we already downloaded all the dependencies, there's no need to rebuild everything.
+# Set this if using Beam 2.37.0 or earlier SDK to speed up job submission.
 ENV PIP_NO_DEPS=True
 
 ENTRYPOINT ["/opt/google/dataflow/python_template_launcher"]
diff --git a/dataflow/flex-templates/streaming_beam/noxfile_config.py b/dataflow/flex-templates/streaming_beam/noxfile_config.py
@@ -25,7 +25,7 @@
     # > ℹ️ We're opting out of all Python versions except 3.8.
     # > The Python version used is defined by the Dockerfile, so it's redundant
     # > to run multiple tests since they would all be running the same Dockerfile.
-    "ignored_versions": ["2.7", "3.6", "3.7", "3.9", "3.10", "3.11"],
+    "ignored_versions": ["2.7", "3.6", "3.7", "3.9", "3.10", "3.11", "3.12"],
     # Old samples are opted out of enforcing Python type hints
     # All new samples should feature them
     "enforce_type_hints": True,
