Skip to content

Commit 5ca8803

Browse files
ohmayrohmayr
authored
fix(templates): resolve core dependencies locally and batch pip installs (#17032)
fix(templates): resolve core dependencies locally and batch pip installs Previously, the `core_deps_from_source` and `prerelease_deps` nox sessions installed core sibling packages sequentially. In a monorepo environment, this caused two major issues: 1. Correctness: It frequently fetched remote upstream code (or PyPI releases) instead of testing the actual local code modified in the active PR. 2. Performance: Sequential `pip install` commands triggered strict dependency resolution repeatedly, leading to severe CI timeouts. This updates the noxfile template to: * Dynamically resolve first-party dependencies from the local `packages/` directory to guarantee the presubmit tests the active PR's code. * Batch the `pip install` commands using `--no-deps` and `--ignore-installed` to bypass the resolver overhead and eliminate the sequential network bottleneck. * Dynamically sort packages in `prerelease_deps` into local vs. PyPI deployments using a regex parser, ensuring safe handling of complex version bounds (e.g., `grpcio>=1.75.1`) without hardcoding multiple lists.
1 parent 43dfc16 commit 5ca8803

9 files changed

Lines changed: 523 additions & 226 deletions

File tree

‎packages/gapic-generator/gapic/templates/noxfile.py.j2‎

Lines changed: 59 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -509,24 +509,51 @@ def prerelease_deps(session, protobuf_implementation):
509509
"proto-plus",
510510
]
511511

512-
for dep in prerel_deps:
513-
session.install("--pre", "--no-deps", "--ignore-installed", dep)
514-
# TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status``
515-
# to the dictionary below once this bug is fixed.
516-
# TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add
517-
# `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below
518-
# once this bug is fixed.
519-
package_namespaces = {
520-
"google-api-core": "google.api_core",
521-
"google-auth": "google.auth",
522-
"grpcio": "grpc",
523-
"protobuf": "google.protobuf",
524-
"proto-plus": "proto",
525-
}
526-
527-
version_namespace = package_namespaces.get(dep)
528-
512+
deps_dir = CURRENT_DIRECTORY.parent
513+
while deps_dir.name != "packages" and deps_dir.parent != deps_dir:
514+
deps_dir = deps_dir.parent
515+
516+
# Extract the base package name, safely ignoring version bounds and spaces
517+
# (e.g., "grpcio>=1.75.1" becomes "grpcio")
518+
parsed_deps = {
519+
dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1)
520+
for dep in prerel_deps
521+
}
522+
523+
# Dynamically sort local packages vs PyPI dependencies
524+
local_paths = []
525+
pypi_deps = []
526+
527+
for dep, pkg_name in parsed_deps.items():
528+
if (deps_dir / pkg_name).exists():
529+
local_paths.append(str(deps_dir / pkg_name))
530+
else:
531+
pypi_deps.append(dep)
532+
533+
# Batch pip installations to avoid sequential overhead
534+
if local_paths:
535+
session.install(*local_paths, "--no-deps", "--ignore-installed")
536+
if pypi_deps:
537+
session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed")
538+
539+
# TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status``
540+
# to the dictionary below once this bug is fixed.
541+
# TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add
542+
# `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below
543+
# once this bug is fixed.
544+
package_namespaces = {
545+
"google-api-core": "google.api_core",
546+
"google-auth": "google.auth",
547+
"grpcio": "grpc",
548+
"protobuf": "google.protobuf",
549+
"proto-plus": "proto",
550+
}
551+
552+
# Reuse the parsed names for logging and version verification
553+
for dep, pkg_name in parsed_deps.items():
529554
print(f"Installed {dep}")
555+
version_namespace = package_namespaces.get(pkg_name)
556+
530557
if version_namespace:
531558
session.run(
532559
"python",
@@ -595,17 +622,23 @@ def core_deps_from_source(session, protobuf_implementation):
595622
# added to the list below so that it is installed from source, rather than PyPI
596623
# Note: If a dependency is added to the `core_dependencies_from_source` list,
597624
# the `prerel_deps` list in the `prerelease_deps` nox session should also be updated.
598-
core_dependencies_from_source = [
599-
"googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos",
600-
"google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core",
601-
"google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth",
602-
"grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1",
603-
"proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus",
625+
core_dependencies_from_source = [
626+
"googleapis-common-protos",
627+
"google-api-core",
628+
"google-auth",
629+
"grpc-google-iam-v1",
630+
"proto-plus",
604631
]
605632

606-
for dep in core_dependencies_from_source:
607-
session.install(dep, "--no-deps", "--ignore-installed")
608-
print(f"Installed {dep}")
633+
deps_dir = CURRENT_DIRECTORY.parent
634+
while deps_dir.name != "packages" and deps_dir.parent != deps_dir:
635+
deps_dir = deps_dir.parent
636+
637+
# Batch the pip installation to avoid sequential overhead
638+
dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source]
639+
640+
session.install(*dep_paths, "--no-deps", "--ignore-installed")
641+
print(f"Installed {', '.join(core_dependencies_from_source)} locally from {deps_dir}")
609642

610643
session.run(
611644
"py.test",

‎packages/gapic-generator/tests/integration/goldens/asset/noxfile.py‎

Lines changed: 58 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -501,24 +501,51 @@ def prerelease_deps(session, protobuf_implementation):
501501
"proto-plus",
502502
]
503503

504-
for dep in prerel_deps:
505-
session.install("--pre", "--no-deps", "--ignore-installed", dep)
506-
# TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status``
507-
# to the dictionary below once this bug is fixed.
508-
# TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add
509-
# `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below
510-
# once this bug is fixed.
511-
package_namespaces = {
512-
"google-api-core": "google.api_core",
513-
"google-auth": "google.auth",
514-
"grpcio": "grpc",
515-
"protobuf": "google.protobuf",
516-
"proto-plus": "proto",
517-
}
518-
519-
version_namespace = package_namespaces.get(dep)
520-
504+
deps_dir = CURRENT_DIRECTORY.parent
505+
while deps_dir.name != "packages" and deps_dir.parent != deps_dir:
506+
deps_dir = deps_dir.parent
507+
508+
# Extract the base package name, safely ignoring version bounds and spaces
509+
# (e.g., "grpcio>=1.75.1" becomes "grpcio")
510+
parsed_deps = {
511+
dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1)
512+
for dep in prerel_deps
513+
}
514+
515+
# Dynamically sort local packages vs PyPI dependencies
516+
local_paths = []
517+
pypi_deps = []
518+
519+
for dep, pkg_name in parsed_deps.items():
520+
if (deps_dir / pkg_name).exists():
521+
local_paths.append(str(deps_dir / pkg_name))
522+
else:
523+
pypi_deps.append(dep)
524+
525+
# Batch pip installations to avoid sequential overhead
526+
if local_paths:
527+
session.install(*local_paths, "--no-deps", "--ignore-installed")
528+
if pypi_deps:
529+
session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed")
530+
531+
# TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status``
532+
# to the dictionary below once this bug is fixed.
533+
# TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add
534+
# `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below
535+
# once this bug is fixed.
536+
package_namespaces = {
537+
"google-api-core": "google.api_core",
538+
"google-auth": "google.auth",
539+
"grpcio": "grpc",
540+
"protobuf": "google.protobuf",
541+
"proto-plus": "proto",
542+
}
543+
544+
# Reuse the parsed names for logging and version verification
545+
for dep, pkg_name in parsed_deps.items():
521546
print(f"Installed {dep}")
547+
version_namespace = package_namespaces.get(pkg_name)
548+
522549
if version_namespace:
523550
session.run(
524551
"python",
@@ -588,16 +615,22 @@ def core_deps_from_source(session, protobuf_implementation):
588615
# Note: If a dependency is added to the `core_dependencies_from_source` list,
589616
# the `prerel_deps` list in the `prerelease_deps` nox session should also be updated.
590617
core_dependencies_from_source = [
591-
"googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos",
592-
"google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core",
593-
"google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth",
594-
"grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1",
595-
"proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus",
618+
"googleapis-common-protos",
619+
"google-api-core",
620+
"google-auth",
621+
"grpc-google-iam-v1",
622+
"proto-plus",
596623
]
597624

598-
for dep in core_dependencies_from_source:
599-
session.install(dep, "--no-deps", "--ignore-installed")
600-
print(f"Installed {dep}")
625+
deps_dir = CURRENT_DIRECTORY.parent
626+
while deps_dir.name != "packages" and deps_dir.parent != deps_dir:
627+
deps_dir = deps_dir.parent
628+
629+
# Batch the pip installation to avoid sequential overhead
630+
dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source]
631+
632+
session.install(*dep_paths, "--no-deps", "--ignore-installed")
633+
print(f"Installed {', '.join(core_dependencies_from_source)} locally from {deps_dir}")
601634

602635
session.run(
603636
"py.test",

‎packages/gapic-generator/tests/integration/goldens/credentials/noxfile.py‎

Lines changed: 58 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -501,24 +501,51 @@ def prerelease_deps(session, protobuf_implementation):
501501
"proto-plus",
502502
]
503503

504-
for dep in prerel_deps:
505-
session.install("--pre", "--no-deps", "--ignore-installed", dep)
506-
# TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status``
507-
# to the dictionary below once this bug is fixed.
508-
# TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add
509-
# `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below
510-
# once this bug is fixed.
511-
package_namespaces = {
512-
"google-api-core": "google.api_core",
513-
"google-auth": "google.auth",
514-
"grpcio": "grpc",
515-
"protobuf": "google.protobuf",
516-
"proto-plus": "proto",
517-
}
518-
519-
version_namespace = package_namespaces.get(dep)
520-
504+
deps_dir = CURRENT_DIRECTORY.parent
505+
while deps_dir.name != "packages" and deps_dir.parent != deps_dir:
506+
deps_dir = deps_dir.parent
507+
508+
# Extract the base package name, safely ignoring version bounds and spaces
509+
# (e.g., "grpcio>=1.75.1" becomes "grpcio")
510+
parsed_deps = {
511+
dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1)
512+
for dep in prerel_deps
513+
}
514+
515+
# Dynamically sort local packages vs PyPI dependencies
516+
local_paths = []
517+
pypi_deps = []
518+
519+
for dep, pkg_name in parsed_deps.items():
520+
if (deps_dir / pkg_name).exists():
521+
local_paths.append(str(deps_dir / pkg_name))
522+
else:
523+
pypi_deps.append(dep)
524+
525+
# Batch pip installations to avoid sequential overhead
526+
if local_paths:
527+
session.install(*local_paths, "--no-deps", "--ignore-installed")
528+
if pypi_deps:
529+
session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed")
530+
531+
# TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status``
532+
# to the dictionary below once this bug is fixed.
533+
# TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add
534+
# `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below
535+
# once this bug is fixed.
536+
package_namespaces = {
537+
"google-api-core": "google.api_core",
538+
"google-auth": "google.auth",
539+
"grpcio": "grpc",
540+
"protobuf": "google.protobuf",
541+
"proto-plus": "proto",
542+
}
543+
544+
# Reuse the parsed names for logging and version verification
545+
for dep, pkg_name in parsed_deps.items():
521546
print(f"Installed {dep}")
547+
version_namespace = package_namespaces.get(pkg_name)
548+
522549
if version_namespace:
523550
session.run(
524551
"python",
@@ -588,16 +615,22 @@ def core_deps_from_source(session, protobuf_implementation):
588615
# Note: If a dependency is added to the `core_dependencies_from_source` list,
589616
# the `prerel_deps` list in the `prerelease_deps` nox session should also be updated.
590617
core_dependencies_from_source = [
591-
"googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos",
592-
"google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core",
593-
"google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth",
594-
"grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1",
595-
"proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus",
618+
"googleapis-common-protos",
619+
"google-api-core",
620+
"google-auth",
621+
"grpc-google-iam-v1",
622+
"proto-plus",
596623
]
597624

598-
for dep in core_dependencies_from_source:
599-
session.install(dep, "--no-deps", "--ignore-installed")
600-
print(f"Installed {dep}")
625+
deps_dir = CURRENT_DIRECTORY.parent
626+
while deps_dir.name != "packages" and deps_dir.parent != deps_dir:
627+
deps_dir = deps_dir.parent
628+
629+
# Batch the pip installation to avoid sequential overhead
630+
dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source]
631+
632+
session.install(*dep_paths, "--no-deps", "--ignore-installed")
633+
print(f"Installed {', '.join(core_dependencies_from_source)} locally from {deps_dir}")
601634

602635
session.run(
603636
"py.test",

0 commit comments

Comments
 (0)