diff --git a/.github/scripts/hermetic_library_generation.sh b/.github/scripts/hermetic_library_generation.sh index 73f4a39426..2c5552f77d 100755 --- a/.github/scripts/hermetic_library_generation.sh +++ b/.github/scripts/hermetic_library_generation.sh @@ -84,16 +84,29 @@ git show "${target_branch}":"${generation_config}" > "${baseline_generation_conf # get .m2 folder so it's mapped into the docker container m2_folder=$(dirname "$(mvn help:evaluate -Dexpression=settings.localRepository -q -DforceStdout)") +# download api definitions from googleapis repository +googleapis_commitish=$(grep googleapis_commitish "${generation_config}" | cut -d ":" -f 2 | xargs) +api_def_dir=$(mktemp -d) +git clone https://github.com/googleapis/googleapis.git "${api_def_dir}" +pushd "${api_def_dir}" +git checkout "${googleapis_commitish}" +popd + # run hermetic code generation docker image. docker run \ --rm \ -u "$(id -u):$(id -g)" \ -v "$(pwd):${workspace_name}" \ -v "${m2_folder}":/home/.m2 \ + -v "${api_def_dir}:${workspace_name}/googleapis" \ -e GENERATOR_VERSION="${image_tag}" \ gcr.io/cloud-devrel-public-resources/java-library-generation:"${image_tag}" \ --baseline-generation-config-path="${workspace_name}/${baseline_generation_config}" \ - --current-generation-config-path="${workspace_name}/${generation_config}" + --current-generation-config-path="${workspace_name}/${generation_config}" \ + --api-definitions-path="${workspace_name}/googleapis" + +# remove api definitions after generation +rm -rf "${api_def_dir}" # commit the change to the pull request. rm -rdf output googleapis "${baseline_generation_config}" diff --git a/library_generation/DEVELOPMENT.md b/library_generation/DEVELOPMENT.md index 858b467797..cc893133ed 100644 --- a/library_generation/DEVELOPMENT.md +++ b/library_generation/DEVELOPMENT.md @@ -101,8 +101,14 @@ shell session. ## Running the script The entrypoint script (`library_generation/cli/entry_point.py`) allows you to -update the target repository with the latest changes starting from the -googleapis committish declared in `generation_config.yaml`. +generate a GAPIC repository with a given api definition (proto, service yaml). + +### Download the api definition +For example, googleapis +``` +git clone https://github.com/googleapis/googleapis +export api_definitions_path="$(pwd)/googleapis" +``` ### Download the repo For example, google-cloud-java @@ -118,7 +124,9 @@ python -m pip install . ### Run the script ``` -python cli/entry_point.py generate --repository-path="${path_to_repo}" +python cli/entry_point.py generate \ + --repository-path="${path_to_repo}" \ + --api-definitions-path="${api_definitions_path}" ``` @@ -144,16 +152,25 @@ repo to this folder). To run the docker container on the google-cloud-java repo, you must run: ```bash -docker run -u "$(id -u)":"$(id -g)" -v/path/to/google-cloud-java:/workspace $(cat image-id) +docker run \ + -u "$(id -u)":"$(id -g)" \ + -v /path/to/google-cloud-java:/workspace \ + -v /path/to/api-definition:/workspace/apis \ + $(cat image-id) \ + --api-definitions-path=/workspace/apis ``` * `-u "$(id -u)":"$(id -g)"` makes docker run the container impersonating yourself. This avoids folder ownership changes since it runs as root by default. - * `-v/path/to/google-cloud-java:/workspace` maps the host machine's - google-cloud-java folder to the /workspace folder. The image is configured to - perform changes in this directory - * `$(cat image-id)` obtains the image ID created in the build step + * `-v /path/to/google-cloud-java:/workspace` maps the host machine's + google-cloud-java folder to the /workspace folder. + The image is configured to perform changes in this directory. + * `-v /path/to/api-definition:/workspace/apis` maps the host machine's + api-definition folder to /workspace/apis folder. + * `$(cat image-id)` obtains the image ID created in the build step. + * `--api-definitions-path=/workspace/apis` set the API definition path to + `/workspace/apis`. ## Debug the created containers If you are working on changing the way the containers are created, you may want @@ -173,5 +190,10 @@ We add `less` and `vim` as text tools for further inspection. You can also run a shell in a new container by running: ```bash -docker run --rm -it -u=$(id -u):$(id -g) -v/path/to/google-cloud-java:/workspace --entrypoint="bash" $(cat image-id) +docker run \ + --rm -it \ + -u $(id -u):$(id -g) \ + -v /path/to/google-cloud-java:/workspace \ + --entrypoint="bash" \ + $(cat image-id) ``` diff --git a/library_generation/README.md b/library_generation/README.md index bead08e290..3496568288 100644 --- a/library_generation/README.md +++ b/library_generation/README.md @@ -1,7 +1,7 @@ # Generate a repository containing GAPIC Client Libraries The script, `entry_point.py`, allows you to generate a repository containing -GAPIC client libraries with googleapis commit history (a monorepo, for example, +GAPIC client libraries with change history (a monorepo, for example, google-cloud-java) from a configuration file. ## Environment @@ -48,6 +48,22 @@ right version for each library. Please refer [here](go/java-client-releasing#versionstxt-manifest) for more info of versions.txt. +### Api definitions path (`api_definitions_path`), optional + +The path to where the api definition (proto, service yaml) resides. + +The default value is the current working directory when running the script. + +Note that you need not only the protos defined the service, but also the transitive +dependencies of those protos. +Any missing dependencies will cause `File not found` error. + +For example, if your service is defined in `example_service.proto` and it imports +`google/api/annotations.proto`, you need the `annotations.proto` resides in a +folder that has the exact structure of the import statement (`google/api` in this +case), and set `api_definitions_path` to the path contains the root folder (`google` +in this case). + ## Output of `entry_point.py` ### GAPIC libraries @@ -74,11 +90,13 @@ will be created/modified: | pom.xml (repo root dir) | Always generated from inputs | | versions.txt | New entries will be added if they don’t exist | -### googleapis commit history +### Change history If both `baseline_generation_config` and `current_generation_config` are -specified, and they contain different googleapis commit, the commit history will -be generated into `pr_description.txt` in the `repository_path`. +specified and the contents are different, the changed contents will be generated +into `pr_description.txt` in the `repository_path`. +In addition, if the `googleapis_commitish` is different, the googleapis commit +history will be generated. ## Configuration to generate a repository @@ -96,7 +114,7 @@ They are shared by library level parameters. | gapic_generator_version | No | set through env variable if not specified | | protoc_version | No | inferred from the generator if not specified | | grpc_version | No | inferred from the generator if not specified | -| googleapis-commitish | Yes | | +| googleapis_commitish | Yes | | | libraries_bom_version | No | empty string if not specified | ### Library level parameters @@ -183,14 +201,16 @@ The virtual environment can be installed to any folder, usually it is recommende 2. Assuming the virtual environment is installed under `sdk-platform-java`. Run the following command under the root folder of `sdk-platform-java` to install the dependencies of `library_generation` -```bash -python -m pip install -r library_generation/requirements.txt -``` + ```bash + python -m pip install -r library_generation/requirements.txt + ``` 3. Run the following command to install `library_generation` as a module, which allows the `library_generation` module to be imported from anywhere -```bash -python -m pip install library_generation/ -``` + ```bash + python -m pip install library_generation/ + ``` + +4. Download api definition to a local directory ## An example to generate a repository using `entry_point.py` @@ -198,7 +218,8 @@ python -m pip install library_generation/ python library_generation/entry_point.py generate \ --baseline-generation-config-path=/path/to/baseline_config_file \ --current-generation-config-path=/path/to/current_config_file \ ---repository-path=path/to/repository +--repository-path=path/to/repository \ +--api-definitions-path=path/to/api_definition ``` If you run `entry_point.py` with the example [configuration](#an-example-of-generation-configuration) shown above, the repository structure is: diff --git a/library_generation/cli/entry_point.py b/library_generation/cli/entry_point.py index d69dde1b77..e5055d8030 100644 --- a/library_generation/cli/entry_point.py +++ b/library_generation/cli/entry_point.py @@ -63,10 +63,22 @@ def main(ctx): directory. """, ) +@click.option( + "--api-definitions-path", + type=str, + default=".", + show_default=True, + help=""" + The path to which the api definition (proto and service yaml) and its + dependencies resides. + If not specified, the path is the current working directory. + """, +) def generate( baseline_generation_config_path: str, current_generation_config_path: str, repository_path: str, + api_definitions_path: str, ): """ Compare baseline generation config and current generation config and @@ -90,7 +102,10 @@ def generate( repository_path/pr_description.txt. """ __generate_repo_and_pr_description_impl( - baseline_generation_config_path, current_generation_config_path, repository_path + baseline_generation_config_path=baseline_generation_config_path, + current_generation_config_path=current_generation_config_path, + repository_path=repository_path, + api_definitions_path=api_definitions_path, ) @@ -98,6 +113,7 @@ def __generate_repo_and_pr_description_impl( baseline_generation_config_path: str, current_generation_config_path: str, repository_path: str, + api_definitions_path: str, ): """ Implementation method for generate(). @@ -129,6 +145,7 @@ def __generate_repo_and_pr_description_impl( current_generation_config_path = os.path.abspath(current_generation_config_path) repository_path = os.path.abspath(repository_path) + api_definitions_path = os.path.abspath(api_definitions_path) if not baseline_generation_config_path: # Execute full generation based on current_generation_config if # baseline_generation_config is not specified. @@ -136,6 +153,7 @@ def __generate_repo_and_pr_description_impl( generate_from_yaml( config=from_yaml(current_generation_config_path), repository_path=repository_path, + api_definitions_path=api_definitions_path, ) return @@ -155,6 +173,7 @@ def __generate_repo_and_pr_description_impl( generate_from_yaml( config=config_change.current_config, repository_path=repository_path, + api_definitions_path=api_definitions_path, target_library_names=target_library_names, ) generate_pr_descriptions( diff --git a/library_generation/generate_composed_library.py b/library_generation/generate_composed_library.py index 2fd014f791..51d1ef21f3 100755 --- a/library_generation/generate_composed_library.py +++ b/library_generation/generate_composed_library.py @@ -59,10 +59,6 @@ def generate_composed_library( :return None """ output_folder = repo_config.output_folder - util.pull_api_definition( - config=config, library=library, output_folder=output_folder - ) - base_arguments = __construct_tooling_arg(config=config) owlbot_cli_source_folder = util.sh_util("mktemp -d") os.makedirs(f"{library_path}", exist_ok=True) @@ -73,7 +69,7 @@ def generate_composed_library( # generate postprocessing prerequisite files (.repo-metadata.json, .OwlBot-hermetic.yaml, # owlbot.py) here because transport is parsed from BUILD.bazel, # which lives in a versioned proto_path. The value of transport will be - # overriden by the config object if specified. Note that this override + # overridden by the config object if specified. Note that this override # does not affect library generation but instead used only for # generating postprocessing files such as README. util.generate_postprocessing_prerequisite_files( diff --git a/library_generation/generate_repo.py b/library_generation/generate_repo.py index a4f6a5382e..cb44fadcbe 100755 --- a/library_generation/generate_repo.py +++ b/library_generation/generate_repo.py @@ -12,6 +12,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import os +import shutil + import library_generation.utils.utilities as util from library_generation.generate_composed_library import generate_composed_library from library_generation.model.generation_config import GenerationConfig @@ -22,6 +25,7 @@ def generate_from_yaml( config: GenerationConfig, repository_path: str, + api_definitions_path: str, target_library_names: list[str] = None, ) -> None: """ @@ -31,6 +35,7 @@ def generate_from_yaml( :param config: a GenerationConfig object. :param repository_path: The repository path to which the generated files will be sent. + :param api_definitions_path: The path to where the api definition resides. :param target_library_names: a list of libraries to be generated. If specified, only the library whose library_name is in target_library_names will be generated. @@ -43,6 +48,8 @@ def generate_from_yaml( repo_config = util.prepare_repo( gen_config=config, library_config=target_libraries, repo_path=repository_path ) + # copy api definition to output folder. + shutil.copytree(api_definitions_path, repo_config.output_folder, dirs_exist_ok=True) for library_path, library in repo_config.get_libraries().items(): print(f"generating library {library.get_library_name()}") diff --git a/library_generation/test/cli/entry_point_unit_tests.py b/library_generation/test/cli/entry_point_unit_tests.py index e05c64de55..171be31236 100644 --- a/library_generation/test/cli/entry_point_unit_tests.py +++ b/library_generation/test/cli/entry_point_unit_tests.py @@ -105,9 +105,13 @@ def test_generate_non_monorepo_without_changes_triggers_full_generation( baseline_generation_config_path=config_path, current_generation_config_path=config_path, repository_path=".", + api_definitions_path=".", ) generate_from_yaml.assert_called_with( - config=ANY, repository_path=ANY, target_library_names=None + config=ANY, + repository_path=ANY, + api_definitions_path=ANY, + target_library_names=None, ) @patch("library_generation.cli.entry_point.generate_from_yaml") @@ -134,9 +138,13 @@ def test_generate_non_monorepo_with_changes_triggers_full_generation( baseline_generation_config_path=baseline_config_path, current_generation_config_path=current_config_path, repository_path=".", + api_definitions_path=".", ) generate_from_yaml.assert_called_with( - config=ANY, repository_path=ANY, target_library_names=None + config=ANY, + repository_path=ANY, + api_definitions_path=ANY, + target_library_names=None, ) @patch("library_generation.cli.entry_point.generate_from_yaml") @@ -160,9 +168,13 @@ def test_generate_monorepo_with_common_protos_triggers_full_generation( baseline_generation_config_path=config_path, current_generation_config_path=config_path, repository_path=".", + api_definitions_path=".", ) generate_from_yaml.assert_called_with( - config=ANY, repository_path=ANY, target_library_names=None + config=ANY, + repository_path=ANY, + api_definitions_path=ANY, + target_library_names=None, ) @patch("library_generation.cli.entry_point.generate_from_yaml") @@ -187,7 +199,11 @@ def test_generate_monorepo_without_common_protos_does_not_trigger_full_generatio baseline_generation_config_path=config_path, current_generation_config_path=config_path, repository_path=".", + api_definitions_path=".", ) generate_from_yaml.assert_called_with( - config=ANY, repository_path=ANY, target_library_names=[] + config=ANY, + repository_path=ANY, + api_definitions_path=ANY, + target_library_names=[], ) diff --git a/library_generation/test/integration_tests.py b/library_generation/test/integration_tests.py index ebb8c66afa..c5f8221cc9 100644 --- a/library_generation/test/integration_tests.py +++ b/library_generation/test/integration_tests.py @@ -13,6 +13,7 @@ # limitations under the License. import difflib import json +import tempfile from filecmp import cmp from filecmp import dircmp from git import Repo @@ -40,13 +41,13 @@ repo_prefix = "https://github.com/googleapis" output_dir = shell_call("get_output_folder") # this map tells which branch of each repo should we use for our diff tests -committish_map = { +commitish_map = { "google-cloud-java": "chore/test-hermetic-build", "java-bigtable": "chore/test-hermetic-build", } baseline_config_name = "baseline_generation_config.yaml" current_config_name = "current_generation_config.yaml" - +googleapis_commitish = "113a378d5aad5018876ec0a8cbfd4d6a4f746809" # This variable is used to override the jar created by building the image # with our own downloaded jar in order to lock the integration test to use # a constant version specified in @@ -54,7 +55,7 @@ # This allows us to decouple the generation workflow testing with what the # generator jar will actually generate. # See library_generation/DEVELOPMENT.md ("The Hermetic Build's -# well-known folder). +# well-known folder"). WELL_KNOWN_GENERATOR_JAR_FILENAME = "gapic-generator-java.jar" @@ -70,6 +71,7 @@ def setUp(cls) -> None: os.makedirs(f"{golden_dir}", exist_ok=True) def test_entry_point_running_in_container(self): + api_definitions_path = self.__copy_api_definition(googleapis_commitish) config_files = self.__get_config_files(config_dir) for repo, config_file in config_files: config = from_yaml(config_file) @@ -77,7 +79,7 @@ def test_entry_point_running_in_container(self): config_location = f"{golden_dir}/../{repo}" # 1. pull repository repo_dest = self.__pull_repo_to( - Path(repo_location), repo, committish_map[repo] + Path(repo_location), repo, commitish_map[repo] ) # 2. prepare golden files library_names = self.__get_library_names_from_config(config) @@ -90,6 +92,7 @@ def test_entry_point_running_in_container(self): config_location=config_location, baseline_config=baseline_config_name, current_config=current_config_name, + api_definition=api_definitions_path, ) # 4. compare generation result with golden files print( @@ -188,10 +191,27 @@ def test_entry_point_running_in_container(self): ) print(" PR description comparison succeed.") self.__remove_generated_files() + shutil.rmtree(api_definitions_path) + + @classmethod + def __copy_api_definition(cls, committish: str) -> str: + repo_dest = cls.__pull_repo_to( + dest=tempfile.mkdtemp(), repo="googleapis", committish=committish + ) + api_temp_dir = tempfile.mkdtemp() + print(f"Copying api definition to {api_temp_dir}...") + shutil.copytree( + f"{repo_dest}/google", f"{api_temp_dir}/google", dirs_exist_ok=True + ) + shutil.copytree( + f"{repo_dest}/grafeas", f"{api_temp_dir}/grafeas", dirs_exist_ok=True + ) + shutil.rmtree(repo_dest) + return api_temp_dir @classmethod def __build_image(cls, docker_file: str, cwd: str): - # we build the docker image without removing intermediate containers so + # we build the docker image without removing intermediate containers, so # we can re-test more quickly subprocess.check_call( ["docker", "build", "-f", docker_file, "-t", image_tag, "."], @@ -278,6 +298,7 @@ def __run_entry_point_in_docker_container( config_location: str, baseline_config: str, current_config: str, + api_definition: str, ): # we use the calling user to prevent the mapped volumes from changing # owners @@ -295,12 +316,15 @@ def __run_entry_point_in_docker_container( "-v", f"{config_location}:/workspace/config", "-v", + f"{api_definition}:/workspace/api", + "-v", f"{config_dir}/{WELL_KNOWN_GENERATOR_JAR_FILENAME}:/home/.library_generation/{WELL_KNOWN_GENERATOR_JAR_FILENAME}", "-w", "/workspace/repo", image_tag, f"--baseline-generation-config-path=/workspace/config/{baseline_config}", f"--current-generation-config-path=/workspace/config/{current_config}", + f"--api-definitions-path=/workspace/api", ], ) diff --git a/library_generation/utils/utilities.py b/library_generation/utils/utilities.py index 1aa60d21c0..894a06ec02 100755 --- a/library_generation/utils/utilities.py +++ b/library_generation/utils/utilities.py @@ -171,42 +171,6 @@ def prepare_repo( ) -def pull_api_definition( - config: GenerationConfig, library: LibraryConfig, output_folder: str -) -> None: - """ - Pull APIs definition from googleapis/googleapis repository. - To avoid duplicated pulling, only perform pulling if the library uses a - different commitish than in generation config. - :param config: a GenerationConfig object representing a parsed configuration - yaml - :param library: a LibraryConfig object contained inside config, passed here - for convenience and to prevent all libraries to be processed - :param output_folder: the folder to which APIs definition (proto files) goes - :return: None - """ - googleapis_commitish = config.googleapis_commitish - if library.googleapis_commitish: - googleapis_commitish = library.googleapis_commitish - print(f"using library-specific googleapis commitish: {googleapis_commitish}") - else: - print(f"using common googleapis_commitish: {config.googleapis_commitish}") - - if googleapis_commitish != config.googleapis_commitish: - print("removing existing APIs definition") - shutil.rmtree(f"{output_folder}/google", ignore_errors=True) - shutil.rmtree(f"{output_folder}/grafeas", ignore_errors=True) - - if not ( - os.path.exists(f"{output_folder}/google") - and os.path.exists(f"{output_folder}/grafeas") - ): - print("downloading googleapis") - sh_util( - f"download_googleapis_files_and_folders {output_folder} {googleapis_commitish}" - ) - - def generate_postprocessing_prerequisite_files( config: GenerationConfig, library: LibraryConfig,