From 4e02b0deac6122e9f19b934cb7302d7a00172c12 Mon Sep 17 00:00:00 2001 From: yoshi-automation Date: Tue, 22 Sep 2020 05:58:30 -0700 Subject: [PATCH 1/5] chore(py-library): update decrypt secrets file * chore(py-library): update decrypt secrets file From https://github.com/GoogleCloudPlatform/python-docs-samples/blob/master/scripts/decrypt-secrets.sh * docs: explain conditional Co-authored-by: Jeffrey Rennie Source-Author: Bu Sun Kim <8822365+busunkim96@users.noreply.github.com> Source-Date: Tue Sep 8 11:35:59 2020 -0600 Source-Repo: googleapis/synthtool Source-Sha: d302f93d7f47e2852e585ac35ab2d15585717ec0 Source-Link: https://github.com/googleapis/synthtool/commit/d302f93d7f47e2852e585ac35ab2d15585717ec0 --- scripts/decrypt-secrets.sh | 15 ++++++++++++++- synth.metadata | 6 +++--- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/scripts/decrypt-secrets.sh b/scripts/decrypt-secrets.sh index ff599eb2a..21f6d2a26 100755 --- a/scripts/decrypt-secrets.sh +++ b/scripts/decrypt-secrets.sh @@ -20,14 +20,27 @@ ROOT=$( dirname "$DIR" ) # Work from the project root. cd $ROOT +# Prevent it from overriding files. +# We recommend that sample authors use their own service account files and cloud project. +# In that case, they are supposed to prepare these files by themselves. +if [[ -f "testing/test-env.sh" ]] || \ + [[ -f "testing/service-account.json" ]] || \ + [[ -f "testing/client-secrets.json" ]]; then + echo "One or more target files exist, aborting." + exit 1 +fi + # Use SECRET_MANAGER_PROJECT if set, fallback to cloud-devrel-kokoro-resources. PROJECT_ID="${SECRET_MANAGER_PROJECT:-cloud-devrel-kokoro-resources}" gcloud secrets versions access latest --secret="python-docs-samples-test-env" \ + --project="${PROJECT_ID}" \ > testing/test-env.sh gcloud secrets versions access latest \ --secret="python-docs-samples-service-account" \ + --project="${PROJECT_ID}" \ > testing/service-account.json gcloud secrets versions access latest \ --secret="python-docs-samples-client-secrets" \ - > testing/client-secrets.json \ No newline at end of file + --project="${PROJECT_ID}" \ + > testing/client-secrets.json diff --git a/synth.metadata b/synth.metadata index efee17785..d3a8bb962 100644 --- a/synth.metadata +++ b/synth.metadata @@ -4,7 +4,7 @@ "git": { "name": ".", "remote": "https://github.com/googleapis/python-bigquery.git", - "sha": "a125160696d1453b04a66c967819f90e70e03a52" + "sha": "b716e1c8ecd90142b498b95e7f8830835529cf4a" } }, { @@ -19,14 +19,14 @@ "git": { "name": "synthtool", "remote": "https://github.com/googleapis/synthtool.git", - "sha": "32c758f11b8c578f515a746c9d263b82a615a77c" + "sha": "d302f93d7f47e2852e585ac35ab2d15585717ec0" } }, { "git": { "name": "synthtool", "remote": "https://github.com/googleapis/synthtool.git", - "sha": "32c758f11b8c578f515a746c9d263b82a615a77c" + "sha": "d302f93d7f47e2852e585ac35ab2d15585717ec0" } } ], From a856a6960aaff64c06011c630c685a9b73fc2dc9 Mon Sep 17 00:00:00 2001 From: yoshi-automation Date: Tue, 22 Sep 2020 05:58:44 -0700 Subject: [PATCH 2/5] chore(python-library): use sphinx 1.5.5 for the docfx job Originally tested at: https://github.com/googleapis/python-texttospeech/pull/89 This change will fix the missing docstring in the yaml files. Source-Author: Takashi Matsuo Source-Date: Thu Sep 10 04:12:14 2020 +0000 Source-Repo: googleapis/synthtool Source-Sha: ffcee7952b74f647cbb3ef021d95422f10816fca Source-Link: https://github.com/googleapis/synthtool/commit/ffcee7952b74f647cbb3ef021d95422f10816fca --- docs/conf.py | 2 +- synth.metadata | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 155606c97..ea9830b34 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -29,7 +29,7 @@ # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. -needs_sphinx = "1.6.3" +needs_sphinx = "1.5.5" # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom diff --git a/synth.metadata b/synth.metadata index d3a8bb962..e72c6e2b9 100644 --- a/synth.metadata +++ b/synth.metadata @@ -19,14 +19,14 @@ "git": { "name": "synthtool", "remote": "https://github.com/googleapis/synthtool.git", - "sha": "d302f93d7f47e2852e585ac35ab2d15585717ec0" + "sha": "ffcee7952b74f647cbb3ef021d95422f10816fca" } }, { "git": { "name": "synthtool", "remote": "https://github.com/googleapis/synthtool.git", - "sha": "d302f93d7f47e2852e585ac35ab2d15585717ec0" + "sha": "ffcee7952b74f647cbb3ef021d95422f10816fca" } } ], From ea49d4950b3c681b8c2a4dbc03f2fc336b3f4feb Mon Sep 17 00:00:00 2001 From: yoshi-automation Date: Tue, 22 Sep 2020 05:59:23 -0700 Subject: [PATCH 3/5] =?UTF-8?q?build(python):=20use=20release-publish=20ap?= =?UTF-8?q?p=20for=20notifying=20GitHub=20of=20releas=E2=80=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * build(python): use release-publish app for notifying GitHub of release status * fix: re-add pypi password Source-Author: Bu Sun Kim <8822365+busunkim96@users.noreply.github.com> Source-Date: Wed Sep 16 08:46:42 2020 -0600 Source-Repo: googleapis/synthtool Source-Sha: 257fda18168bedb76985024bd198ed1725485488 Source-Link: https://github.com/googleapis/synthtool/commit/257fda18168bedb76985024bd198ed1725485488 --- .kokoro/release/common.cfg | 50 ++++++++++---------------------------- synth.metadata | 4 +-- 2 files changed, 15 insertions(+), 39 deletions(-) diff --git a/.kokoro/release/common.cfg b/.kokoro/release/common.cfg index 661a04481..18b417709 100644 --- a/.kokoro/release/common.cfg +++ b/.kokoro/release/common.cfg @@ -23,42 +23,18 @@ env_vars: { value: "github/python-bigquery/.kokoro/release.sh" } -# Fetch the token needed for reporting release status to GitHub -before_action { - fetch_keystore { - keystore_resource { - keystore_config_id: 73713 - keyname: "yoshi-automation-github-key" - } - } -} - -# Fetch PyPI password -before_action { - fetch_keystore { - keystore_resource { - keystore_config_id: 73713 - keyname: "google_cloud_pypi_password" - } - } -} - -# Fetch magictoken to use with Magic Github Proxy -before_action { - fetch_keystore { - keystore_resource { - keystore_config_id: 73713 - keyname: "releasetool-magictoken" - } - } +# Fetch PyPI password +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73713 + keyname: "google_cloud_pypi_password" + } + } } -# Fetch api key to use with Magic Github Proxy -before_action { - fetch_keystore { - keystore_resource { - keystore_config_id: 73713 - keyname: "magic-github-proxy-api-key" - } - } -} +# Tokens needed to report release status back to GitHub +env_vars: { + key: "SECRET_MANAGER_KEYS" + value: "releasetool-publish-reporter-app,releasetool-publish-reporter-googleapis-installation,releasetool-publish-reporter-pem" +} \ No newline at end of file diff --git a/synth.metadata b/synth.metadata index e72c6e2b9..d6a25cf2e 100644 --- a/synth.metadata +++ b/synth.metadata @@ -19,14 +19,14 @@ "git": { "name": "synthtool", "remote": "https://github.com/googleapis/synthtool.git", - "sha": "ffcee7952b74f647cbb3ef021d95422f10816fca" + "sha": "257fda18168bedb76985024bd198ed1725485488" } }, { "git": { "name": "synthtool", "remote": "https://github.com/googleapis/synthtool.git", - "sha": "ffcee7952b74f647cbb3ef021d95422f10816fca" + "sha": "257fda18168bedb76985024bd198ed1725485488" } } ], From 4fcd64cd7303889f31606d55dcefe5b4d61106a6 Mon Sep 17 00:00:00 2001 From: yoshi-automation Date: Tue, 22 Sep 2020 05:59:36 -0700 Subject: [PATCH 4/5] build(python): add secret manager in kokoro Source-Author: Bu Sun Kim <8822365+busunkim96@users.noreply.github.com> Source-Date: Wed Sep 16 10:24:40 2020 -0600 Source-Repo: googleapis/synthtool Source-Sha: dba48bb9bc6959c232bec9150ac6313b608fe7bd Source-Link: https://github.com/googleapis/synthtool/commit/dba48bb9bc6959c232bec9150ac6313b608fe7bd --- .kokoro/populate-secrets.sh | 43 +++++++++++++++++++++++++++++++++++++ .kokoro/trampoline.sh | 15 ++++++++----- synth.metadata | 5 +++-- 3 files changed, 56 insertions(+), 7 deletions(-) create mode 100755 .kokoro/populate-secrets.sh diff --git a/.kokoro/populate-secrets.sh b/.kokoro/populate-secrets.sh new file mode 100755 index 000000000..f52514257 --- /dev/null +++ b/.kokoro/populate-secrets.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# Copyright 2020 Google LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eo pipefail + +function now { date +"%Y-%m-%d %H:%M:%S" | tr -d '\n' ;} +function msg { println "$*" >&2 ;} +function println { printf '%s\n' "$(now) $*" ;} + + +# Populates requested secrets set in SECRET_MANAGER_KEYS from service account: +# kokoro-trampoline@cloud-devrel-kokoro-resources.iam.gserviceaccount.com +SECRET_LOCATION="${KOKORO_GFILE_DIR}/secret_manager" +msg "Creating folder on disk for secrets: ${SECRET_LOCATION}" +mkdir -p ${SECRET_LOCATION} +for key in $(echo ${SECRET_MANAGER_KEYS} | sed "s/,/ /g") +do + msg "Retrieving secret ${key}" + docker run --entrypoint=gcloud \ + --volume=${KOKORO_GFILE_DIR}:${KOKORO_GFILE_DIR} \ + gcr.io/google.com/cloudsdktool/cloud-sdk \ + secrets versions access latest \ + --project cloud-devrel-kokoro-resources \ + --secret ${key} > \ + "${SECRET_LOCATION}/${key}" + if [[ $? == 0 ]]; then + msg "Secret written to ${SECRET_LOCATION}/${key}" + else + msg "Error retrieving secret ${key}" + fi +done diff --git a/.kokoro/trampoline.sh b/.kokoro/trampoline.sh index e8c4251f3..f39236e94 100755 --- a/.kokoro/trampoline.sh +++ b/.kokoro/trampoline.sh @@ -15,9 +15,14 @@ set -eo pipefail -python3 "${KOKORO_GFILE_DIR}/trampoline_v1.py" || ret_code=$? +# Always run the cleanup script, regardless of the success of bouncing into +# the container. +function cleanup() { + chmod +x ${KOKORO_GFILE_DIR}/trampoline_cleanup.sh + ${KOKORO_GFILE_DIR}/trampoline_cleanup.sh + echo "cleanup"; +} +trap cleanup EXIT -chmod +x ${KOKORO_GFILE_DIR}/trampoline_cleanup.sh -${KOKORO_GFILE_DIR}/trampoline_cleanup.sh || true - -exit ${ret_code} +$(dirname $0)/populate-secrets.sh # Secret Manager secrets. +python3 "${KOKORO_GFILE_DIR}/trampoline_v1.py" \ No newline at end of file diff --git a/synth.metadata b/synth.metadata index d6a25cf2e..8d95be8d5 100644 --- a/synth.metadata +++ b/synth.metadata @@ -19,14 +19,14 @@ "git": { "name": "synthtool", "remote": "https://github.com/googleapis/synthtool.git", - "sha": "257fda18168bedb76985024bd198ed1725485488" + "sha": "dba48bb9bc6959c232bec9150ac6313b608fe7bd" } }, { "git": { "name": "synthtool", "remote": "https://github.com/googleapis/synthtool.git", - "sha": "257fda18168bedb76985024bd198ed1725485488" + "sha": "dba48bb9bc6959c232bec9150ac6313b608fe7bd" } } ], @@ -60,6 +60,7 @@ ".kokoro/docs/common.cfg", ".kokoro/docs/docs-presubmit.cfg", ".kokoro/docs/docs.cfg", + ".kokoro/populate-secrets.sh", ".kokoro/presubmit/common.cfg", ".kokoro/presubmit/presubmit.cfg", ".kokoro/presubmit/system-2.7.cfg", From 202ca5a680cbb5070322aaf36a3b5755b8b73d89 Mon Sep 17 00:00:00 2001 From: yoshi-automation Date: Tue, 22 Sep 2020 06:00:03 -0700 Subject: [PATCH 5/5] chore(python): add sphinx doctest extension Source-Author: Bu Sun Kim <8822365+busunkim96@users.noreply.github.com> Source-Date: Mon Sep 21 13:09:57 2020 -0600 Source-Repo: googleapis/synthtool Source-Sha: 27f4406999b1eee29e04b09b2423a8e4646c7e24 Source-Link: https://github.com/googleapis/synthtool/commit/27f4406999b1eee29e04b09b2423a8e4646c7e24 --- docs/conf.py | 1 + synth.metadata | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index ea9830b34..b38bdd1ff 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -39,6 +39,7 @@ "sphinx.ext.autosummary", "sphinx.ext.intersphinx", "sphinx.ext.coverage", + "sphinx.ext.doctest", "sphinx.ext.napoleon", "sphinx.ext.todo", "sphinx.ext.viewcode", diff --git a/synth.metadata b/synth.metadata index 8d95be8d5..e786fb364 100644 --- a/synth.metadata +++ b/synth.metadata @@ -19,14 +19,14 @@ "git": { "name": "synthtool", "remote": "https://github.com/googleapis/synthtool.git", - "sha": "dba48bb9bc6959c232bec9150ac6313b608fe7bd" + "sha": "27f4406999b1eee29e04b09b2423a8e4646c7e24" } }, { "git": { "name": "synthtool", "remote": "https://github.com/googleapis/synthtool.git", - "sha": "dba48bb9bc6959c232bec9150ac6313b608fe7bd" + "sha": "27f4406999b1eee29e04b09b2423a8e4646c7e24" } } ],