Python compat (GoogleCloudPlatform#174)

Jon Donovan · Douglas Greiman · commit 01a4da0bc4a4 · 2018-01-12T12:43:31.000-08:00
* Add support for python-compat

* Update app.yaml and tests

* Add missing .dockerignore file

* Updating per review comments

* Adding missing files

* Update gen_dockerfile.py

* Update gen_dockerfile_test.py
diff --git a/scripts/data/Dockerfile.python_compat b/scripts/data/Dockerfile.python_compat
@@ -0,0 +1,3 @@
+FROM gcr.io/google_appengine/python-compat-multicore
+ADD . /app/
+RUN if [ -s requirements.txt ]; then pip install -r requirements.txt; fi
diff --git a/scripts/data/dockerignore.python_compat b/scripts/data/dockerignore.python_compat
@@ -0,0 +1,5 @@
+.dockerignore
+Dockerfile
+.git
+.hg
+.svn
diff --git a/scripts/gen_dockerfile.py b/scripts/gen_dockerfile.py
@@ -29,7 +29,6 @@
 
 import validation_utils
 
-
 # Validate characters for dockerfile image names.
 #
 # This roots out obvious mistakes, the full gory details are here:
@@ -63,7 +62,7 @@
 # Validated application configuration
 AppConfig = collections.namedtuple(
     'AppConfig',
-    'base_image dockerfile_python_version entrypoint has_requirements_txt'
+    'base_image dockerfile_python_version entrypoint has_requirements_txt is_python_compat'
 )
 
 
@@ -97,6 +96,16 @@ def get_app_config(raw_config, base_image, config_file, source_dir):
             'Expected {} contents to be a Mapping type, but found type "{}"'.
             format(config_file, type(raw_config)))
 
+    # Short circuit for python compat.
+    if validation_utils.get_field_value(
+        raw_config, 'runtime', str) == 'python-compat':
+      return AppConfig(
+          base_image=None,
+          dockerfile_python_version=None,
+          entrypoint=None,
+          has_requirements_txt=None,
+          is_python_compat=True)
+
     entrypoint = validation_utils.get_field_value(
         raw_config, 'entrypoint', str)
     if not PRINTABLE_REGEX.match(entrypoint):
@@ -133,7 +142,8 @@ def get_app_config(raw_config, base_image, config_file, source_dir):
         base_image=base_image,
         dockerfile_python_version=dockerfile_python_version,
         entrypoint=entrypoint,
-        has_requirements_txt=has_requirements_txt)
+        has_requirements_txt=has_requirements_txt,
+        is_python_compat=False)
 
 
 def get_data(name):
@@ -175,19 +185,24 @@ def generate_files(app_config):
     else:
         optional_entrypoint = ''
 
-    dockerfile = ''.join([
-        get_data('Dockerfile.preamble.template').format(
-            base_image=app_config.base_image),
-        get_data('Dockerfile.virtualenv.template').format(
-            python_version=app_config.dockerfile_python_version),
-        optional_requirements_txt,
-        get_data('Dockerfile.install_app'),
-        optional_entrypoint,
-    ])
+    if app_config.is_python_compat:
+      dockerfile = get_data('Dockerfile.python_compat')
+      dockerignore = get_data('dockerignore.python_compat')
+    else:
+      dockerfile = ''.join([
+          get_data('Dockerfile.preamble.template').format(
+              base_image=app_config.base_image),
+          get_data('Dockerfile.virtualenv.template').format(
+              python_version=app_config.dockerfile_python_version),
+          optional_requirements_txt,
+          get_data('Dockerfile.install_app'),
+          optional_entrypoint,
+      ])
+      dockerignore =  get_data('dockerignore')
 
     return {
         'Dockerfile': dockerfile,
-        '.dockerignore': get_data('dockerignore'),
+        '.dockerignore': dockerignore,
     }
 
 
@@ -206,7 +221,7 @@ def generate_dockerfile_command(base_image, config_file, source_dir):
 
     # Determine complete configuration
     app_config = get_app_config(raw_config, base_image, config_file,
-                                source_dir)
+                                  source_dir)
 
     # Generate list of filenames and their textual contents
     files = generate_files(app_config)
diff --git a/scripts/gen_dockerfile_test.py b/scripts/gen_dockerfile_test.py
@@ -54,6 +54,14 @@ def compare_file(filename, dir1, dir2):
         'dockerfile_python_version': '',
         'has_requirements_txt': False,
         'entrypoint': '',
+        'is_python_compat': False,
+    }),
+    ('env: flex\nruntime: python-compat', {
+        'base_image': None,
+        'dockerfile_python_version': None,
+        'has_requirements_txt': None,
+        'entrypoint': None,
+        'is_python_compat': True,
     }),
     # All supported python versions
     ('runtime_config:\n python_version:', {
@@ -125,7 +133,8 @@ def test_get_app_config_invalid(app_yaml):
     base_image='',
     dockerfile_python_version='',
     entrypoint='',
-    has_requirements_txt=False
+    has_requirements_txt=False,
+    is_python_compat=False,
 )
 
 
@@ -146,6 +155,9 @@ def test_get_app_config_invalid(app_yaml):
     # Python version
     (_BASE_APP_CONFIG._replace(dockerfile_python_version='_my_version'), True,
      'python_version=python_my_version'),
+    # python-compat runtime
+    (_BASE_APP_CONFIG._replace(is_python_compat=True), True,
+     'FROM gcr.io/google_appengine/python-compat-multicore'),
 ])
 def test_generate_files(app_config, should_find, test_string):
     result = gen_dockerfile.generate_files(app_config)
@@ -163,10 +175,13 @@ def compare_against_golden_files(app, config_dir, testdata_dir):
         compare_file(filename, config_dir, golden_dir)
 
 
-def test_generate_dockerfile_command(tmpdir, testdata_dir):
+@pytest.mark.parametrize('app', [
+    # Sampled from https://github.com/GoogleCloudPlatform/python-docs-samples
+    'hello_world',
+    # From an internal source.
+    'hello_world_compat'])
+def test_generate_dockerfile_command(tmpdir, testdata_dir, app):
     """Generates output and compares against a set of golden files."""
-    # Sample from https://github.com/GoogleCloudPlatform/python-docs-samples
-    app = 'hello_world'
     app_dir = os.path.join(testdata_dir, app)
 
     # Copy sample app to writable temp dir, and generate Dockerfile.
@@ -179,12 +194,15 @@ def test_generate_dockerfile_command(tmpdir, testdata_dir):
     compare_against_golden_files(app, config_dir, testdata_dir)
 
 
+@pytest.mark.parametrize('app', [
+    # Sampled from https://github.com/GoogleCloudPlatform/python-docs-samples
+    'hello_world',
+    # From an internal source.
+    'hello_world_compat'])
 @pytest.mark.xfail(not shutil.which('gcloud'),
                    reason='Google Cloud SDK is not installed')
-def test_generate_dockerfile_golden(tmpdir, testdata_dir):
+def test_generate_dockerfile_golden(tmpdir, testdata_dir, app):
     """Validate our golden files against gcloud app gen-config"""
-    # Sample from https://github.com/GoogleCloudPlatform/python-docs-samples
-    app = 'hello_world'
     app_dir = os.path.join(testdata_dir, app)
 
     # Copy sample app to writable temp dir, and generate Dockerfile.
diff --git a/scripts/testdata/hello_world_compat/app.yaml b/scripts/testdata/hello_world_compat/app.yaml
@@ -0,0 +1,13 @@
+service: default
+runtime: python-compat
+env: flex
+
+api_version: 1
+threadsafe: true
+
+beta_settings:
+  enable_app_engine_apis: true # Needed for compat apps.
+
+handlers:
+- url: .*
+  script: main.app
diff --git a/scripts/testdata/hello_world_compat/main.py b/scripts/testdata/hello_world_compat/main.py
@@ -0,0 +1,14 @@
+"""The hello world flex app!"""
+
+import webapp2
+
+
+class HelloHandler(webapp2.RequestHandler):
+
+  def get(self):
+    msg = 'Hello GAE Flex (env: flex) Compat-Runtime App\n'
+    self.response.headers['Content-Type'] = 'text/plain'
+    self.response.out.write(msg)
+
+app = webapp2.WSGIApplication([('/', HelloHandler)],
+                              debug=True)
diff --git a/scripts/testdata/hello_world_compat_golden/.dockerignore b/scripts/testdata/hello_world_compat_golden/.dockerignore
@@ -0,0 +1,5 @@
+.dockerignore
+Dockerfile
+.git
+.hg
+.svn
diff --git a/scripts/testdata/hello_world_compat_golden/Dockerfile b/scripts/testdata/hello_world_compat_golden/Dockerfile
@@ -0,0 +1,3 @@
+FROM gcr.io/google_appengine/python-compat-multicore
+ADD . /app/
+RUN if [ -s requirements.txt ]; then pip install -r requirements.txt; fi

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+FROM gcr.io/google_appengine/python-compat-multicore`
	`2`	`+ADD . /app/`
	`3`	`+RUN if [ -s requirements.txt ]; then pip install -r requirements.txt; fi`
-Original file line number
+Diff line change
@@ @@ -0,0 +1,5 @@ @@
 +.dockerignore
 +Dockerfile
 +.git
 +.hg
 +.svn