Skip to content

Commit e616863

Browse files
chingor13chingor13
authored
chore(java): use separate autosynth job for README (#767)
Split java README generation into a separate, per-repo continuous job running from Kokoro. We now generate a new job that runs on commits to the primary branch that strictly manages the README.md. This should prevent us from seeing the contextless autosynth PRs which are caused by non-proto/template upstream changes.
1 parent da29da3 commit e616863

7 files changed

Lines changed: 175 additions & 23 deletions

File tree

‎autosynth/synth.py‎

Lines changed: 21 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -374,11 +374,11 @@ def synthesize_loop(
374374
youngest = len(toolbox.versions) - 1
375375
has_changes = toolbox.synthesize_version_in_new_branch(synthesizer, youngest)
376376
if not has_changes:
377-
if not toolbox.metadata_contains_generated_files(
378-
toolbox.branch
379-
) and toolbox.metadata_contains_generated_files(
380-
toolbox.sub_branch(youngest)
381-
) and not change_pusher.check_if_pr_already_exists(toolbox.branch):
377+
if (
378+
not toolbox.metadata_contains_generated_files(toolbox.branch)
379+
and toolbox.metadata_contains_generated_files(toolbox.sub_branch(youngest))
380+
and not change_pusher.check_if_pr_already_exists(toolbox.branch)
381+
):
382382
# Special case: the repo owner turned on obsolete file tracking.
383383
# Generate a one-time PR containing only metadata changes.
384384
executor.check_call(["git", "checkout", toolbox.branch])
@@ -553,7 +553,16 @@ def _inner_main(temp_dir: str) -> int:
553553
parser.add_argument(
554554
"--repository", default=os.environ.get("REPOSITORY"), required=True
555555
)
556-
parser.add_argument("--synth-path", default=os.environ.get("SYNTH_PATH"))
556+
parser.add_argument(
557+
"--synth-path",
558+
default=os.environ.get("SYNTH_PATH"),
559+
help="If specified, changes the directory from which synthtool is invoked.",
560+
)
561+
parser.add_argument(
562+
"--synth-file-name",
563+
default=os.environ.get("SYNTH_FILE_NAME"),
564+
help="If specified, override the synth file name and may be a path to a file. Defaults to 'synth.py'.",
565+
)
557566
parser.add_argument("--metadata-path", default=os.environ.get("METADATA_PATH"))
558567
parser.add_argument("--base-log-dir", default="")
559568
parser.add_argument(
@@ -579,6 +588,7 @@ def _inner_main(temp_dir: str) -> int:
579588
f"the API or client library generator."
580589
)
581590
change_pusher: AbstractChangePusher = ChangePusher(args.repository, gh, branch)
591+
synth_file_name = args.synth_file_name or "synth.py"
582592

583593
# capture logs for later
584594
# The logs directory path will be rendered in Sponge and Fusion as the test name,
@@ -629,6 +639,7 @@ def _inner_main(temp_dir: str) -> int:
629639
metadata_path,
630640
args.extra_args,
631641
deprecated_execution=args.deprecated_execution,
642+
synth_py_path=synth_file_name,
632643
).synthesize(synth_log_path / "sponge_log.log")
633644

634645
if not has_changes():
@@ -655,7 +666,10 @@ def _inner_main(temp_dir: str) -> int:
655666

656667
# Prepare to call synthesize loop.
657668
synthesizer = Synthesizer(
658-
metadata_path, args.extra_args, args.deprecated_execution, "synth.py",
669+
metadata_path,
670+
args.extra_args,
671+
deprecated_execution=args.deprecated_execution,
672+
synth_py_path=synth_file_name,
659673
)
660674
x = SynthesizeLoopToolbox(
661675
source_versions,

‎synthtool/gcp/common.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -296,7 +296,9 @@ def ruby_library(self, **kwargs) -> Path:
296296
return self._generic_library("ruby_library", **kwargs)
297297

298298
def render(self, template_name: str, **kwargs) -> Path:
299-
return self._templates.render(template_name, **kwargs)
299+
template = self._templates.render(template_name, **kwargs)
300+
_tracked_paths.add(template)
301+
return template
300302

301303
def _load_generic_metadata(self, metadata: Dict):
302304
"""

‎synthtool/gcp/templates/java_library/.github/readme/synth.py‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
# Copyright 2020 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
"""This script is used to synthesize generated the README for this library."""
16+
17+
from synthtool.languages import java
18+
19+
java.custom_templates(["java_library/README.md"])

‎synthtool/gcp/templates/java_library/.kokoro/continuous/readme.cfg‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
# Copyright 2020 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
# Format: //devtools/kokoro/config/proto/build.proto
16+
17+
env_vars: {
18+
key: "TRAMPOLINE_IMAGE"
19+
value: "gcr.io/cloud-devrel-kokoro-resources/python-multi"
20+
}
21+
22+
env_vars: {
23+
key: "TRAMPOLINE_BUILD_FILE"
24+
value: "github/{{ metadata['repo']['repo_short'] }}/.kokoro/readme.sh"
25+
}
26+
27+
# Build logs will be here
28+
action {
29+
define_artifacts {
30+
regex: "**/*sponge_log.xml"
31+
regex: "**/*sponge_log.log"
32+
}
33+
}
34+
35+
# The github token is stored here.
36+
before_action {
37+
fetch_keystore {
38+
keystore_resource {
39+
keystore_config_id: 73713
40+
keyname: "yoshi-automation-github-key"
41+
# TODO(theacodes): remove this after secrets have globally propagated
42+
backend_type: FASTCONFIGPUSH
43+
}
44+
}
45+
}
46+
47+
# Common env vars for all repositories and builds.
48+
env_vars: {
49+
key: "GITHUB_USER"
50+
value: "yoshi-automation"
51+
}
52+
env_vars: {
53+
key: "GITHUB_EMAIL"
54+
value: "yoshi-automation@google.com"
55+
}

‎synthtool/gcp/templates/java_library/.kokoro/readme.sh‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
#!/bin/bash
2+
# Copyright 2020 Google LLC
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
set -eo pipefail
17+
18+
cd ${KOKORO_ARTIFACTS_DIR}/github/{{ metadata['repo']['repo_short'] }}
19+
20+
# Disable buffering, so that the logs stream through.
21+
export PYTHONUNBUFFERED=1
22+
23+
# Kokoro exposes this as a file, but the scripts expect just a plain variable.
24+
export GITHUB_TOKEN=$(cat ${KOKORO_KEYSTORE_DIR}/73713_yoshi-automation-github-key)
25+
26+
# Setup git credentials
27+
echo "https://${GITHUB_TOKEN}:@github.com" >> ~/.git-credentials
28+
git config --global credential.helper 'store --file ~/.git-credentials'
29+
30+
python3.6 -m pip install git+https://github.com/googleapis/synthtool.git#egg=gcp-synthtool
31+
python3.6 -m autosynth.synth \
32+
--repository={{metadata['repo']['repo']}} \
33+
--synth-file-name=.github/readme/synth.py \
34+
--metadata-path=.github/readme/synth.metadata \
35+
--pr-title="chore: regenerate README" \
36+
--branch-suffix="readme"

‎synthtool/languages/java.py‎

Lines changed: 40 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -323,19 +323,7 @@ def _merge_common_templates(
323323
return source_text
324324

325325

326-
def common_templates(
327-
excludes: List[str] = [], template_path: Optional[Path] = None, **kwargs
328-
) -> None:
329-
"""Generate common templates for a Java Library
330-
331-
Fetches information about the repository from the .repo-metadata.json file,
332-
information about the latest artifact versions and copies the files into
333-
their expected location.
334-
335-
Args:
336-
excludes (List[str], optional): List of template paths to ignore
337-
**kwargs: Additional options for CommonTemplates.java_library()
338-
"""
326+
def _common_template_metadata() -> Dict[str, Any]:
339327
metadata = {} # type: Dict[str, Any]
340328
repo_metadata = common._load_repo_metadata()
341329
if repo_metadata:
@@ -359,6 +347,44 @@ def common_templates(
359347
else:
360348
metadata["min_java_version"] = DEFAULT_MIN_SUPPORTED_JAVA_VERSION
361349

362-
kwargs["metadata"] = metadata
350+
return metadata
351+
352+
353+
def common_templates(
354+
excludes: List[str] = [], template_path: Optional[Path] = None, **kwargs
355+
) -> None:
356+
"""Generate common templates for a Java Library
357+
358+
Fetches information about the repository from the .repo-metadata.json file,
359+
information about the latest artifact versions and copies the files into
360+
their expected location.
361+
362+
Args:
363+
excludes (List[str], optional): List of template paths to ignore
364+
**kwargs: Additional options for CommonTemplates.java_library()
365+
"""
366+
kwargs["metadata"] = _common_template_metadata()
363367
templates = gcp.CommonTemplates(template_path=template_path).java_library(**kwargs)
368+
369+
# README.md is now synthesized separately. This prevents synthtool from deleting the
370+
# README as it's no longer generated here.
371+
excludes.append("README.md")
372+
364373
s.copy([templates], excludes=excludes, merge=_merge_common_templates)
374+
375+
376+
def custom_templates(files: List[str], **kwargs) -> None:
377+
"""Generate custom template files
378+
379+
Fetches information about the repository from the .repo-metadata.json file,
380+
information about the latest artifact versions and copies the files into
381+
their expected location.
382+
383+
Args:
384+
files (List[str], optional): List of template paths to include
385+
**kwargs: Additional options for CommonTemplates.render()
386+
"""
387+
kwargs["metadata"] = _common_template_metadata()
388+
for file in files:
389+
template = gcp.CommonTemplates().render(file, **kwargs)
390+
s.copy([template])

‎tests/test_language_java.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,7 @@ def assert_valid_yaml(file):
100100
try:
101101
# generate the common templates
102102
java.common_templates(template_path=TEMPLATES_PATH)
103-
assert os.path.isfile("README.md")
103+
assert os.path.isfile("renovate.json")
104104

105105
# lint xml, yaml files
106106
# use os.walk because glob ignores hidden directories

0 commit comments

Comments
 (0)