Update kubernetespodoperator (GoogleCloudPlatform#2270)

leahecole · web-flow · commit 9095b4b6362d · 2019-07-15T10:15:54.000-07:00
* Remove one example of a secret, chg param names

* Remove one example of a secret, chg param names

* Secret object region tags

* Secret object region tags

* Min region tag

* Min region tag

* Add more region tags

* Add more region tags

* Move region tag

* Move region tag

* remove region tags`

* Fix linter issues
diff --git a/composer/workflows/kubernetes_pod_operator.py b/composer/workflows/kubernetes_pod_operator.py
@@ -16,44 +16,31 @@
 
 # [START composer_kubernetespodoperator]
 import datetime
-
 from airflow import models
-# [END composer_kubernetespodoperator]
 from airflow.contrib.kubernetes import pod
 from airflow.contrib.kubernetes import secret
-# [START composer_kubernetespodoperator]
 from airflow.contrib.operators import kubernetes_pod_operator
 
-# [END composer_kubernetespodoperator]
 
 # A Secret is an object that contains a small amount of sensitive data such as
 # a password, a token, or a key. Such information might otherwise be put in a
 # Pod specification or in an image; putting it in a Secret object allows for
 # more control over how it is used, and reduces the risk of accidental
 # exposure.
-secret_file = secret.Secret(
-    # Mounts the secret as a file in RAM-backed tmpfs.
-    deploy_type='volume',
-    # File path of where to deploy the target, since deploy_type is 'volume'
-    # rather than 'env'.
-    deploy_target='/etc/sql_conn',
-    # Name of secret in Kubernetes, if the secret is not already defined in
-    # Kubernetes using kubectl the Pod will fail to find the secret, and in
-    # turn, fail to launch.
-    secret='airflow-secrets',
-    # Key of the secret within Kubernetes.
-    key='sql_alchemy_conn')
 
+# [START composer_kubernetespodoperator_secretobject]
 secret_env = secret.Secret(
     # Expose the secret as environment variable.
     deploy_type='env',
     # The name of the environment variable, since deploy_type is `env` rather
     # than `volume`.
     deploy_target='SQL_CONN',
+    # Name of the Kubernetes Secret
     secret='airflow-secrets',
+    # Key of a secret stored in this Secret object
     key='sql_alchemy_conn')
+# [END composer_kubernetespodoperator_secretobject]
 
-# [START composer_kubernetespodoperator]
 YESTERDAY = datetime.datetime.now() - datetime.timedelta(days=1)
 
 # If a Pod fails to launch, or has an error occur in the container, Airflow
@@ -69,6 +56,8 @@
     # no `config_file` parameter is specified. By default it will contain the
     # credentials for Cloud Composer's Google Kubernetes Engine cluster that is
     # created upon environment creation.
+
+    # [START composer_kubernetespodoperator_minconfig]
     kubernetes_min_pod = kubernetes_pod_operator.KubernetesPodOperator(
         # The ID specified for the task.
         task_id='pod-ex-minimum',
@@ -89,8 +78,8 @@
         # gcr.io images if the Composer Environment is under the same
         # project-id as the gcr.io images.
         image='gcr.io/gcp-runtimes/ubuntu_16_0_4')
-    # [END composer_kubernetespodoperator]
-
+    # [END composer_kubernetespodoperator_minconfig]
+    # [START composer_kubernetespodoperator_templateconfig]
     kubenetes_template_ex = kubernetes_pod_operator.KubernetesPodOperator(
         task_id='ex-kube-templates',
         name='ex-kube-templates',
@@ -113,23 +102,26 @@
         # setting the environment variable `MY_VALUE`. The pod will fail if
         # `my_value` is not set in the Airflow UI.
         env_vars={'MY_VALUE': '{{ var.value.my_value }}'},
-        # Sets the config file to the specified airflow.cfg airflow home. If
-        # the configuration file does not exist or does not provide valid
-        # credentials the pod will fail to launch.
-        config_file="{{ conf.get('core', 'airflow_home') }}/config")
-
+        # Sets the config file to a kubernetes config file specified in
+        # airflow.cfg. If the configuration file does not exist or does
+        # not provide validcredentials the pod will fail to launch. If not
+        # specified, config_file defaults to ~/.kube/config
+        config_file="{{ conf.get('core', 'kube_config') }}")
+    # [END composer_kubernetespodoperator_templateconfig]
+    # [START composer_kubernetespodoperator_secretconfig]
     kubernetes_secret_vars_ex = kubernetes_pod_operator.KubernetesPodOperator(
         task_id='ex-kube-secrets',
         name='ex-kube-secrets',
         namespace='default',
         image='ubuntu',
+        startup_timeout_seconds=300,
         # The secrets to pass to Pod, the Pod will fail to create if the
         # secrets you specify in a Secret object do not exist in Kubernetes.
-        secrets=[secret_env, secret_file],
+        secrets=[secret_env],
         # env_vars allows you to specify environment variables for your
         # container to use. env_vars is templated.
         env_vars={'EXAMPLE_VAR': '/example/value'})
-
+    # [END composer_kubernetespodoperator_secretconfig]
     # [START composer_kubernetespodaffinity]
     kubernetes_affinity_ex = kubernetes_pod_operator.KubernetesPodOperator(
         task_id='ex-pod-affinity',
@@ -163,16 +155,16 @@
                             # The label key's value that pods can be scheduled
                             # on.
                             'values': [
-                                'node-pool-name-1',
-                                'node-pool-name-2',
+                                'pool-0',
+                                'pool-1',
                             ]
                         }]
                     }]
                 }
             }
         })
     # [END composer_kubernetespodaffinity]
-
+    # [START composer_kubernetespodoperator_fullconfig]
     kubernetes_full_pod = kubernetes_pod_operator.KubernetesPodOperator(
         task_id='ex-all-configs',
         name='pi',
@@ -221,3 +213,5 @@
         # config. For more information see:
         # https://kubernetes.io/docs/concepts/configuration/assign-pod-node/
         affinity={})
+    # [END composer_kubernetespodoperator_fullconfig]
+    # [END composer_kubernetespodoperator]
