testing: migrate to trampoline_v2 (GoogleCloudPlatform#3860)

Takashi Matsuo · web-flow · commit 89fa5fc60cd6 · 2020-05-29T15:29:11.000-07:00
* testing: prototype trampoline_v2

* add RUN_TESTS_DIRS envvar

* add a script for running tests locally

* don't rely on git, remove noxfile.py only if we copied it

* add TRAMPOLINE_IMAGE_UPLOAD envvar

* add example

* add .trampolinerc_tmpl

* allow docker in docker

* use /workspace for project root

* use $PROJECT_ROOT instead of $PWD

* also pass down KOKORO_KEYSTORE_DIR

* pass down Kokoro dynamic variables

* introduce TRAMPOLINE_CI and TRAMPOLINE_V2 envvar

* use --entrypoint instead of giving the command

* add kbuilder's uid as the default arg

* do not show the docker commands by default

* make the mount point configurable

* change TRAMPOLINE_IMAGE_SOURCE to TRAMPOLINE_DOCKERFILE

* only decrypt the secret when the key exists

* also prevents decrypt-secrets.sh to override existing files.

* remove secret if we used decrypt-secrets.sh before the test

* add 2 more envvars for build cop bot

* setup kokoro service account only on kokoro

* address review comments

* correctly report the exit value

* remove unnecessary build args

* use docker gid

* download cloud_sql_proxy to ${HOME} again
diff --git a/.kokoro/docker/Dockerfile b/.kokoro/docker/Dockerfile
@@ -161,4 +161,19 @@ ENV PATH /google-cloud-sdk/bin:$PATH
 # Enable redis-server on boot.
 RUN sudo systemctl enable redis-server.service
 
+# Create a user and allow sudo
+
+# kbuilder uid on the default Kokoro image
+ARG UID=1000
+ARG USERNAME=kbuilder
+
+# Add a new user to the container image.
+# This is needed for ssh and sudo access.
+
+# Add a new user with the caller's uid and the username.
+RUN useradd -d /h -u ${UID} ${USERNAME}
+
+# Allow nopasswd sudo
+RUN echo "${USERNAME} ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers
+
 CMD ["python3.6"]
diff --git a/.kokoro/lint/common.cfg b/.kokoro/lint/common.cfg
@@ -24,7 +24,7 @@ env_vars: {
 gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline"
 
 # Use the trampoline script to run in docker.
-build_file: "python-docs-samples/.kokoro/trampoline.sh"
+build_file: "python-docs-samples/.kokoro/trampoline_v2.sh"
 
 # Download secrets from Cloud Storage.
 gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples"
@@ -41,3 +41,8 @@ env_vars: {
     key: "RUN_TESTS_SESSION"
     value: "lint"
 }
+
+env_vars: {
+    key: "TRAMPOLINE_DOCKERFILE"
+    value: ".kokoro/docker/Dockerfile"
+}
diff --git a/.kokoro/lint/continuous.cfg b/.kokoro/lint/continuous.cfg
@@ -17,5 +17,5 @@
 # Tell the trampoline which build file to use.
 env_vars: {
     key: "TRAMPOLINE_BUILD_FILE"
-    value: "github/python-docs-samples/.kokoro/tests/run_tests_diff_head.sh"
+    value: ".kokoro/tests/run_tests_diff_head.sh"
 }
diff --git a/.kokoro/lint/periodic.cfg b/.kokoro/lint/periodic.cfg
@@ -17,5 +17,5 @@
 # Tell the trampoline which build file to use.
 env_vars: {
     key: "TRAMPOLINE_BUILD_FILE"
-    value: "github/python-docs-samples/.kokoro/tests/run_tests.sh"
+    value: ".kokoro/tests/run_tests.sh"
 }
diff --git a/.kokoro/lint/presubmit.cfg b/.kokoro/lint/presubmit.cfg
@@ -17,5 +17,5 @@
 # Tell the trampoline which build file to use.
 env_vars: {
     key: "TRAMPOLINE_BUILD_FILE"
-    value: "github/python-docs-samples/.kokoro/tests/run_tests_diff_master.sh"
+    value: ".kokoro/tests/run_tests_diff_master.sh"
 }
diff --git a/.kokoro/python2.7/common.cfg b/.kokoro/python2.7/common.cfg
@@ -26,7 +26,7 @@ env_vars: {
 gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline"
 
 # Use the trampoline script to run in docker.
-build_file: "python-docs-samples/.kokoro/trampoline.sh"
+build_file: "python-docs-samples/.kokoro/trampoline_v2.sh"
 
 # Download secrets from Cloud Storage.
 gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples"
@@ -44,9 +44,13 @@ env_vars: {
     value: "py-2.7"
 }
 
-# Declare build specific Cloud project. It still uses the common one,
-# but we'll update the value once we have more Cloud projects.
+# Declare build specific Cloud project.
 env_vars: {
     key: "BUILD_SPECIFIC_GCLOUD_PROJECT"
     value: "python-docs-samples-tests"
 }
+
+env_vars: {
+    key: "TRAMPOLINE_DOCKERFILE"
+    value: ".kokoro/docker/Dockerfile"
+}
diff --git a/.kokoro/python2.7/continuous.cfg b/.kokoro/python2.7/continuous.cfg
@@ -17,5 +17,5 @@
 # Tell the trampoline which build file to use.
 env_vars: {
     key: "TRAMPOLINE_BUILD_FILE"
-    value: "github/python-docs-samples/.kokoro/tests/run_tests_diff_head.sh"
+    value: ".kokoro/tests/run_tests_diff_head.sh"
 }
diff --git a/.kokoro/python2.7/periodic.cfg b/.kokoro/python2.7/periodic.cfg
@@ -17,10 +17,10 @@
 # Tell the trampoline which build file to use.
 env_vars: {
     key: "TRAMPOLINE_BUILD_FILE"
-    value: "github/python-docs-samples/.kokoro/tests/run_tests.sh"
+    value: ".kokoro/tests/run_tests.sh"
 }
 
 env_vars: {
     key: "REPORT_TO_BUILD_COP_BOT"
-    value: "True"
+    value: "true"
 }
diff --git a/.kokoro/python2.7/presubmit.cfg b/.kokoro/python2.7/presubmit.cfg
@@ -17,5 +17,5 @@
 # Tell the trampoline which build file to use.
 env_vars: {
     key: "TRAMPOLINE_BUILD_FILE"
-    value: "github/python-docs-samples/.kokoro/tests/run_tests_diff_master.sh"
+    value: ".kokoro/tests/run_tests_diff_master.sh"
 }
diff --git a/.kokoro/python3.6/common.cfg b/.kokoro/python3.6/common.cfg
@@ -26,7 +26,7 @@ env_vars: {
 gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline"
 
 # Use the trampoline script to run in docker.
-build_file: "python-docs-samples/.kokoro/trampoline.sh"
+build_file: "python-docs-samples/.kokoro/trampoline_v2.sh"
 
 # Download secrets from Cloud Storage.
 gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples"
@@ -50,3 +50,8 @@ env_vars: {
     key: "BUILD_SPECIFIC_GCLOUD_PROJECT"
     value: "python-docs-samples-tests-py36"
 }
+
+env_vars: {
+    key: "TRAMPOLINE_DOCKERFILE"
+    value: ".kokoro/docker/Dockerfile"
+}
diff --git a/.kokoro/python3.6/continuous.cfg b/.kokoro/python3.6/continuous.cfg
@@ -17,5 +17,5 @@
 # Tell the trampoline which build file to use.
 env_vars: {
     key: "TRAMPOLINE_BUILD_FILE"
-    value: "github/python-docs-samples/.kokoro/tests/run_tests_diff_head.sh"
+    value: ".kokoro/tests/run_tests_diff_head.sh"
 }
diff --git a/.kokoro/python3.6/periodic.cfg b/.kokoro/python3.6/periodic.cfg
@@ -17,10 +17,10 @@
 # Tell the trampoline which build file to use.
 env_vars: {
     key: "TRAMPOLINE_BUILD_FILE"
-    value: "github/python-docs-samples/.kokoro/tests/run_tests.sh"
+    value: ".kokoro/tests/run_tests.sh"
 }
 
 env_vars: {
     key: "REPORT_TO_BUILD_COP_BOT"
-    value: "True"
+    value: "true"
 }
diff --git a/.kokoro/python3.6/presubmit.cfg b/.kokoro/python3.6/presubmit.cfg
@@ -17,5 +17,5 @@
 # Tell the trampoline which build file to use.
 env_vars: {
     key: "TRAMPOLINE_BUILD_FILE"
-    value: "github/python-docs-samples/.kokoro/tests/run_tests_diff_master.sh"
+    value: ".kokoro/tests/run_tests_diff_master.sh"
 }
diff --git a/.kokoro/python3.7/common.cfg b/.kokoro/python3.7/common.cfg
@@ -26,7 +26,7 @@ env_vars: {
 gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline"
 
 # Use the trampoline script to run in docker.
-build_file: "python-docs-samples/.kokoro/trampoline.sh"
+build_file: "python-docs-samples/.kokoro/trampoline_v2.sh"
 
 # Download secrets from Cloud Storage.
 gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples"
@@ -50,3 +50,8 @@ env_vars: {
     key: "BUILD_SPECIFIC_GCLOUD_PROJECT"
     value: "python-docs-samples-tests-py37"
 }
+
+env_vars: {
+    key: "TRAMPOLINE_DOCKERFILE"
+    value: ".kokoro/docker/Dockerfile"
+}
diff --git a/.kokoro/python3.7/continuous.cfg b/.kokoro/python3.7/continuous.cfg
@@ -17,5 +17,5 @@
 # Tell the trampoline which build file to use.
 env_vars: {
     key: "TRAMPOLINE_BUILD_FILE"
-    value: "github/python-docs-samples/.kokoro/tests/run_tests_diff_head.sh"
+    value: ".kokoro/tests/run_tests_diff_head.sh"
 }
diff --git a/.kokoro/python3.7/periodic.cfg b/.kokoro/python3.7/periodic.cfg
@@ -17,10 +17,10 @@
 # Tell the trampoline which build file to use.
 env_vars: {
     key: "TRAMPOLINE_BUILD_FILE"
-    value: "github/python-docs-samples/.kokoro/tests/run_tests.sh"
+    value: ".kokoro/tests/run_tests.sh"
 }
 
 env_vars: {
     key: "REPORT_TO_BUILD_COP_BOT"
-    value: "True"
+    value: "true"
 }
diff --git a/.kokoro/python3.7/presubmit.cfg b/.kokoro/python3.7/presubmit.cfg
@@ -14,8 +14,7 @@
 
 # Format: //devtools/kokoro/config/proto/build.proto
 
-# Tell the trampoline which build file to use.
 env_vars: {
     key: "TRAMPOLINE_BUILD_FILE"
-    value: "github/python-docs-samples/.kokoro/tests/run_tests_diff_master.sh"
+    value: ".kokoro/tests/run_tests_diff_master.sh"
 }
diff --git a/.kokoro/python3.8/common.cfg b/.kokoro/python3.8/common.cfg
@@ -26,7 +26,7 @@ env_vars: {
 gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline"
 
 # Use the trampoline script to run in docker.
-build_file: "python-docs-samples/.kokoro/trampoline.sh"
+build_file: "python-docs-samples/.kokoro/trampoline_v2.sh"
 
 # Download secrets from Cloud Storage.
 gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples"
@@ -50,3 +50,8 @@ env_vars: {
     key: "BUILD_SPECIFIC_GCLOUD_PROJECT"
     value: "python-docs-samples-tests-py38"
 }
+
+env_vars: {
+    key: "TRAMPOLINE_DOCKERFILE"
+    value: ".kokoro/docker/Dockerfile"
+}
diff --git a/.kokoro/python3.8/continuous.cfg b/.kokoro/python3.8/continuous.cfg
@@ -17,5 +17,5 @@
 # Tell the trampoline which build file to use.
 env_vars: {
     key: "TRAMPOLINE_BUILD_FILE"
-    value: "github/python-docs-samples/.kokoro/tests/run_tests_diff_head.sh"
+    value: ".kokoro/tests/run_tests_diff_head.sh"
 }
diff --git a/.kokoro/python3.8/periodic.cfg b/.kokoro/python3.8/periodic.cfg
@@ -17,10 +17,10 @@
 # Tell the trampoline which build file to use.
 env_vars: {
     key: "TRAMPOLINE_BUILD_FILE"
-    value: "github/python-docs-samples/.kokoro/tests/run_tests.sh"
+    value: ".kokoro/tests/run_tests.sh"
 }
 
 env_vars: {
     key: "REPORT_TO_BUILD_COP_BOT"
-    value: "True"
+    value: "true"
 }
diff --git a/.kokoro/python3.8/presubmit.cfg b/.kokoro/python3.8/presubmit.cfg
@@ -17,5 +17,5 @@
 # Tell the trampoline which build file to use.
 env_vars: {
     key: "TRAMPOLINE_BUILD_FILE"
-    value: "github/python-docs-samples/.kokoro/tests/run_tests_diff_master.sh"
+    value: ".kokoro/tests/run_tests_diff_master.sh"
 }
diff --git a/.kokoro/tests/run_tests.sh b/.kokoro/tests/run_tests.sh
diff --git a/.kokoro/trampoline_v2.sh b/.kokoro/trampoline_v2.sh
diff --git a/.trampolinerc b/.trampolinerc
diff --git a/.trampolinerc_tmpl b/.trampolinerc_tmpl
diff --git a/scripts/decrypt-secrets.sh b/scripts/decrypt-secrets.sh
diff --git a/scripts/run_tests_local.sh b/scripts/run_tests_local.sh

Original file line number	Diff line number	Diff line change
`@@ -17,5 +17,5 @@`
`17`	`17`	`# Tell the trampoline which build file to use.`
`18`	`18`	`env_vars: {`
`19`	`19`	`key: "TRAMPOLINE_BUILD_FILE"`
`20`		`- value: "github/python-docs-samples/.kokoro/tests/run_tests_diff_head.sh"`
	`20`	`+ value: ".kokoro/tests/run_tests_diff_head.sh"`
`21`	`21`	`}`
Original file line number	Diff line number	Diff line change
`@@ -17,10 +17,10 @@`
`17`	`17`	`# Tell the trampoline which build file to use.`
`18`	`18`	`env_vars: {`
`19`	`19`	`key: "TRAMPOLINE_BUILD_FILE"`
`20`		`- value: "github/python-docs-samples/.kokoro/tests/run_tests.sh"`
	`20`	`+ value: ".kokoro/tests/run_tests.sh"`
`21`	`21`	`}`
`22`	`22`
`23`	`23`	`env_vars: {`
`24`	`24`	`key: "REPORT_TO_BUILD_COP_BOT"`
`25`		`- value: "True"`
	`25`	`+ value: "true"`
`26`	`26`	`}`