diff --git a/packages/bigquery-magics/noxfile.py b/packages/bigquery-magics/noxfile.py index f275406b6905..42c95921df76 100644 --- a/packages/bigquery-magics/noxfile.py +++ b/packages/bigquery-magics/noxfile.py @@ -455,47 +455,67 @@ def prerelease_deps(session): session.install(*constraints_deps) prerel_deps = [ - "protobuf", - # dependency of grpc - "six", - "grpc-google-iam-v1", "googleapis-common-protos", - "grpcio", - "grpcio-status", "google-api-core", "google-auth", + "grpc-google-iam-v1", + "grpcio>=1.75.1" if session.python >= "3.12" else "grpcio<=1.62.2", + "grpcio-status", + "protobuf", "proto-plus", - "google-cloud-testutils", - # dependencies of google-cloud-testutils" - "click", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--upgrade", dep) - - # Remaining dependencies - other_deps = [ - "requests", - ] - session.install(*other_deps) - - # Install spanner-graph-notebook, python-bigquery, and python-bigquery-storage - # from main to detect any potential breaking changes. For context, see: - # https://github.com/googleapis/python-bigquery-pandas/issues/854 - session.install( - "--ignore-installed", - # TODO(https://github.com/googleapis/google-cloud-python/pull/126): Install this again when we relax the pin. - # "https://github.com/cloudspannerecosystem/spanner-graph-notebook/archive/refs/heads/main.zip", - f"{CURRENT_DIRECTORY}/../google-cloud-bigquery", - f"{CURRENT_DIRECTORY}/../google-cloud-bigquery-storage", - ) - # Print out prerelease package versions - session.run("pip", "freeze") - session.run( - "python", "-c", "import google.protobuf; print(google.protobuf.__version__)" - ) - session.run("python", "-c", "import grpc; print(grpc.__version__)") - session.run("python", "-c", "import google.auth; print(google.auth.__version__)") + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): + print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + + if version_namespace: + session.run( + "python", + "-c", + f"import {version_namespace}; print({version_namespace}.__version__)", + ) session.run( "py.test", diff --git a/packages/gapic-generator/WORKSPACE b/packages/gapic-generator/WORKSPACE index 4da1de3cc7d4..1c0d13fec883 100644 --- a/packages/gapic-generator/WORKSPACE +++ b/packages/gapic-generator/WORKSPACE @@ -123,3 +123,12 @@ switched_rules_by_language( gapic = True, grpc = True, ) + +# BEGIN workaround +load("@rules_python//python:repositories.bzl", "python_register_toolchains") + +python_register_toolchains( + name = "python311", + python_version = "3.11", +) +# END workaround diff --git a/packages/gapic-generator/gapic/templates/noxfile.py.j2 b/packages/gapic-generator/gapic/templates/noxfile.py.j2 index 244173b4b42d..db4a29fcf9db 100644 --- a/packages/gapic-generator/gapic/templates/noxfile.py.j2 +++ b/packages/gapic-generator/gapic/templates/noxfile.py.j2 @@ -509,24 +509,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations to avoid sequential overhead + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -595,17 +622,23 @@ def core_deps_from_source(session, protobuf_implementation): # added to the list below so that it is installed from source, rather than PyPI # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. - core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + core_dependencies_from_source = [ + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] - for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + + session.install(*dep_paths, "--no-deps", "--ignore-installed") + print(f"Installed {', '.join(core_dependencies_from_source)} locally from {deps_dir}") session.run( "py.test", diff --git a/packages/gapic-generator/tests/integration/goldens/asset/noxfile.py b/packages/gapic-generator/tests/integration/goldens/asset/noxfile.py index 654c242376b8..9abef9c30d27 100755 --- a/packages/gapic-generator/tests/integration/goldens/asset/noxfile.py +++ b/packages/gapic-generator/tests/integration/goldens/asset/noxfile.py @@ -501,24 +501,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations to avoid sequential overhead + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -588,16 +615,22 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] - for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + + session.install(*dep_paths, "--no-deps", "--ignore-installed") + print(f"Installed {', '.join(core_dependencies_from_source)} locally from {deps_dir}") session.run( "py.test", diff --git a/packages/gapic-generator/tests/integration/goldens/credentials/noxfile.py b/packages/gapic-generator/tests/integration/goldens/credentials/noxfile.py index 6d4d59fd617e..bfa0326f3845 100755 --- a/packages/gapic-generator/tests/integration/goldens/credentials/noxfile.py +++ b/packages/gapic-generator/tests/integration/goldens/credentials/noxfile.py @@ -501,24 +501,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations to avoid sequential overhead + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -588,16 +615,22 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] - for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + + session.install(*dep_paths, "--no-deps", "--ignore-installed") + print(f"Installed {', '.join(core_dependencies_from_source)} locally from {deps_dir}") session.run( "py.test", diff --git a/packages/gapic-generator/tests/integration/goldens/eventarc/noxfile.py b/packages/gapic-generator/tests/integration/goldens/eventarc/noxfile.py index 37e78b41e841..18dbb71ca4ee 100755 --- a/packages/gapic-generator/tests/integration/goldens/eventarc/noxfile.py +++ b/packages/gapic-generator/tests/integration/goldens/eventarc/noxfile.py @@ -501,24 +501,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations to avoid sequential overhead + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -588,16 +615,22 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] - for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + + session.install(*dep_paths, "--no-deps", "--ignore-installed") + print(f"Installed {', '.join(core_dependencies_from_source)} locally from {deps_dir}") session.run( "py.test", diff --git a/packages/gapic-generator/tests/integration/goldens/logging/noxfile.py b/packages/gapic-generator/tests/integration/goldens/logging/noxfile.py index 4f061e71824c..f7e6f9091aaf 100755 --- a/packages/gapic-generator/tests/integration/goldens/logging/noxfile.py +++ b/packages/gapic-generator/tests/integration/goldens/logging/noxfile.py @@ -501,24 +501,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations to avoid sequential overhead + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -588,16 +615,22 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] - for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + + session.install(*dep_paths, "--no-deps", "--ignore-installed") + print(f"Installed {', '.join(core_dependencies_from_source)} locally from {deps_dir}") session.run( "py.test", diff --git a/packages/gapic-generator/tests/integration/goldens/logging_internal/noxfile.py b/packages/gapic-generator/tests/integration/goldens/logging_internal/noxfile.py index 4f061e71824c..f7e6f9091aaf 100755 --- a/packages/gapic-generator/tests/integration/goldens/logging_internal/noxfile.py +++ b/packages/gapic-generator/tests/integration/goldens/logging_internal/noxfile.py @@ -501,24 +501,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations to avoid sequential overhead + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -588,16 +615,22 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] - for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + + session.install(*dep_paths, "--no-deps", "--ignore-installed") + print(f"Installed {', '.join(core_dependencies_from_source)} locally from {deps_dir}") session.run( "py.test", diff --git a/packages/gapic-generator/tests/integration/goldens/redis/noxfile.py b/packages/gapic-generator/tests/integration/goldens/redis/noxfile.py index c224715196d7..05e6e98292a4 100755 --- a/packages/gapic-generator/tests/integration/goldens/redis/noxfile.py +++ b/packages/gapic-generator/tests/integration/goldens/redis/noxfile.py @@ -501,24 +501,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations to avoid sequential overhead + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -588,16 +615,22 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] - for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + + session.install(*dep_paths, "--no-deps", "--ignore-installed") + print(f"Installed {', '.join(core_dependencies_from_source)} locally from {deps_dir}") session.run( "py.test", diff --git a/packages/gapic-generator/tests/integration/goldens/redis_selective/noxfile.py b/packages/gapic-generator/tests/integration/goldens/redis_selective/noxfile.py index c224715196d7..05e6e98292a4 100755 --- a/packages/gapic-generator/tests/integration/goldens/redis_selective/noxfile.py +++ b/packages/gapic-generator/tests/integration/goldens/redis_selective/noxfile.py @@ -501,24 +501,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations to avoid sequential overhead + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -588,16 +615,22 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] - for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + + session.install(*dep_paths, "--no-deps", "--ignore-installed") + print(f"Installed {', '.join(core_dependencies_from_source)} locally from {deps_dir}") session.run( "py.test", diff --git a/packages/gapic-generator/tests/integration/goldens/storagebatchoperations/noxfile.py b/packages/gapic-generator/tests/integration/goldens/storagebatchoperations/noxfile.py index 02371d0616c5..9dbe1aa24718 100755 --- a/packages/gapic-generator/tests/integration/goldens/storagebatchoperations/noxfile.py +++ b/packages/gapic-generator/tests/integration/goldens/storagebatchoperations/noxfile.py @@ -501,24 +501,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations to avoid sequential overhead + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -588,16 +615,22 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] - for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + + session.install(*dep_paths, "--no-deps", "--ignore-installed") + print(f"Installed {', '.join(core_dependencies_from_source)} locally from {deps_dir}") session.run( "py.test", diff --git a/packages/google-ads-admanager/noxfile.py b/packages/google-ads-admanager/noxfile.py index b3e1290e38e3..0d37120b4301 100644 --- a/packages/google-ads-admanager/noxfile.py +++ b/packages/google-ads-admanager/noxfile.py @@ -510,24 +510,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -597,16 +624,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-ads-datamanager/noxfile.py b/packages/google-ads-datamanager/noxfile.py index f15cd3fda2ff..44ccc4ac6a73 100644 --- a/packages/google-ads-datamanager/noxfile.py +++ b/packages/google-ads-datamanager/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-ads-marketingplatform-admin/noxfile.py b/packages/google-ads-marketingplatform-admin/noxfile.py index 807f7de93864..e7697e28d34a 100644 --- a/packages/google-ads-marketingplatform-admin/noxfile.py +++ b/packages/google-ads-marketingplatform-admin/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-ai-generativelanguage/noxfile.py b/packages/google-ai-generativelanguage/noxfile.py index 47a112449d24..595ef015f880 100644 --- a/packages/google-ai-generativelanguage/noxfile.py +++ b/packages/google-ai-generativelanguage/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-analytics-admin/noxfile.py b/packages/google-analytics-admin/noxfile.py index 06490b7bddd0..12814c834fee 100644 --- a/packages/google-analytics-admin/noxfile.py +++ b/packages/google-analytics-admin/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-analytics-data/noxfile.py b/packages/google-analytics-data/noxfile.py index 91e2913611ac..683b5a8fa782 100644 --- a/packages/google-analytics-data/noxfile.py +++ b/packages/google-analytics-data/noxfile.py @@ -510,24 +510,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -597,16 +624,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-apps-card/noxfile.py b/packages/google-apps-card/noxfile.py index daa345f4d537..8351760c751c 100644 --- a/packages/google-apps-card/noxfile.py +++ b/packages/google-apps-card/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-apps-chat/noxfile.py b/packages/google-apps-chat/noxfile.py index 6d12d58f0aec..a70fe1d958f5 100644 --- a/packages/google-apps-chat/noxfile.py +++ b/packages/google-apps-chat/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-apps-events-subscriptions/noxfile.py b/packages/google-apps-events-subscriptions/noxfile.py index f9a46d3dabe0..5f4788a1759c 100644 --- a/packages/google-apps-events-subscriptions/noxfile.py +++ b/packages/google-apps-events-subscriptions/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-apps-meet/noxfile.py b/packages/google-apps-meet/noxfile.py index e0f6b0fa20e9..68d39de650bf 100644 --- a/packages/google-apps-meet/noxfile.py +++ b/packages/google-apps-meet/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-apps-script-type/noxfile.py b/packages/google-apps-script-type/noxfile.py index 4e3a157a00b9..e7edf74f266a 100644 --- a/packages/google-apps-script-type/noxfile.py +++ b/packages/google-apps-script-type/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-area120-tables/noxfile.py b/packages/google-area120-tables/noxfile.py index 908370f5f6da..415c487c63c5 100644 --- a/packages/google-area120-tables/noxfile.py +++ b/packages/google-area120-tables/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-auth-httplib2/noxfile.py b/packages/google-auth-httplib2/noxfile.py index 67b1bd420cc5..47e2c97157d0 100644 --- a/packages/google-auth-httplib2/noxfile.py +++ b/packages/google-auth-httplib2/noxfile.py @@ -407,21 +407,67 @@ def prerelease_deps(session): session.install(*constraints_deps) prerel_deps = [ + "googleapis-common-protos", + "google-api-core", "google-auth", - "httplib2", + "grpc-google-iam-v1", + "grpcio>=1.75.1" if session.python >= "3.12" else "grpcio<=1.62.2", + "grpcio-status", + "protobuf", + "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--upgrade", dep) - - # Remaining dependencies - other_deps = [ - "requests", - ] - session.install(*other_deps) - - # Print out prerelease package versions - session.run("python", "-c", "import google.auth; print(google.auth.__version__)") + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): + print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + + if version_namespace: + session.run( + "python", + "-c", + f"import {version_namespace}; print({version_namespace}.__version__)", + ) session.run( "py.test", @@ -489,20 +535,24 @@ def core_deps_from_source(session): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - f"{CURRENT_DIRECTORY}/../google-auth", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] - for dep in core_dependencies_from_source: - session.install("--pre", "--no-deps", "--upgrade", dep) + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent - # Remaining dependencies - other_deps = [ - "requests", - ] - session.install(*other_deps) + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") - # Print out prerelease package versions - session.run("python", "-c", "import google.auth; print(google.auth.__version__)") + for dep in core_dependencies_from_source: + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-access-approval/noxfile.py b/packages/google-cloud-access-approval/noxfile.py index d6a8cf47d154..33451b7b78d2 100644 --- a/packages/google-cloud-access-approval/noxfile.py +++ b/packages/google-cloud-access-approval/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-access-context-manager/noxfile.py b/packages/google-cloud-access-context-manager/noxfile.py index b6ae8c821dca..66346440dd03 100644 --- a/packages/google-cloud-access-context-manager/noxfile.py +++ b/packages/google-cloud-access-context-manager/noxfile.py @@ -464,30 +464,57 @@ def prerelease_deps(session, protobuf_implementation): "google-api-core", "google-auth", "grpc-google-iam-v1", - "grpcio", + "grpcio>=1.75.1" if session.python >= "3.12" else "grpcio<=1.62.2", "grpcio-status", "protobuf", "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -559,16 +586,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - f"{CURRENT_DIRECTORY}/../googleapis-common-protos", - f"{CURRENT_DIRECTORY}/../google-api-core", - f"{CURRENT_DIRECTORY}/../google-auth", - f"{CURRENT_DIRECTORY}/../grpc-google-iam-v1", - f"{CURRENT_DIRECTORY}/../proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-advisorynotifications/noxfile.py b/packages/google-cloud-advisorynotifications/noxfile.py index 79a4a374fa34..12deb9b55845 100644 --- a/packages/google-cloud-advisorynotifications/noxfile.py +++ b/packages/google-cloud-advisorynotifications/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-alloydb-connectors/noxfile.py b/packages/google-cloud-alloydb-connectors/noxfile.py index 1720b4b4f31a..13ee32508614 100644 --- a/packages/google-cloud-alloydb-connectors/noxfile.py +++ b/packages/google-cloud-alloydb-connectors/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-alloydb/noxfile.py b/packages/google-cloud-alloydb/noxfile.py index 94a9f19c779e..9309bb1515c7 100644 --- a/packages/google-cloud-alloydb/noxfile.py +++ b/packages/google-cloud-alloydb/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-api-gateway/noxfile.py b/packages/google-cloud-api-gateway/noxfile.py index 35017f69a253..12c661da510a 100644 --- a/packages/google-cloud-api-gateway/noxfile.py +++ b/packages/google-cloud-api-gateway/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-api-keys/noxfile.py b/packages/google-cloud-api-keys/noxfile.py index da7909f1bdc0..1f7aea7d168a 100644 --- a/packages/google-cloud-api-keys/noxfile.py +++ b/packages/google-cloud-api-keys/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-apigee-connect/noxfile.py b/packages/google-cloud-apigee-connect/noxfile.py index 6291c10fd893..18004efc9b71 100644 --- a/packages/google-cloud-apigee-connect/noxfile.py +++ b/packages/google-cloud-apigee-connect/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-apigee-registry/noxfile.py b/packages/google-cloud-apigee-registry/noxfile.py index e63701651122..308b20b9e56a 100644 --- a/packages/google-cloud-apigee-registry/noxfile.py +++ b/packages/google-cloud-apigee-registry/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-apihub/noxfile.py b/packages/google-cloud-apihub/noxfile.py index 40a8f714ca3f..69f8b84adaed 100644 --- a/packages/google-cloud-apihub/noxfile.py +++ b/packages/google-cloud-apihub/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-apiregistry/noxfile.py b/packages/google-cloud-apiregistry/noxfile.py index d9fef843f682..e1d39b429d18 100644 --- a/packages/google-cloud-apiregistry/noxfile.py +++ b/packages/google-cloud-apiregistry/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-appengine-admin/noxfile.py b/packages/google-cloud-appengine-admin/noxfile.py index fd198a77da53..82df6696f88c 100644 --- a/packages/google-cloud-appengine-admin/noxfile.py +++ b/packages/google-cloud-appengine-admin/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-appengine-logging/noxfile.py b/packages/google-cloud-appengine-logging/noxfile.py index 1fcdae83b641..3b32ba9b8e88 100644 --- a/packages/google-cloud-appengine-logging/noxfile.py +++ b/packages/google-cloud-appengine-logging/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-apphub/noxfile.py b/packages/google-cloud-apphub/noxfile.py index 7e38a6a4e554..e6230d10f3d9 100644 --- a/packages/google-cloud-apphub/noxfile.py +++ b/packages/google-cloud-apphub/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-appoptimize/noxfile.py b/packages/google-cloud-appoptimize/noxfile.py index c48652694aee..1de72066a194 100644 --- a/packages/google-cloud-appoptimize/noxfile.py +++ b/packages/google-cloud-appoptimize/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-artifact-registry/noxfile.py b/packages/google-cloud-artifact-registry/noxfile.py index 870b58cc9ce7..9b84d096087c 100644 --- a/packages/google-cloud-artifact-registry/noxfile.py +++ b/packages/google-cloud-artifact-registry/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-asset/noxfile.py b/packages/google-cloud-asset/noxfile.py index 49b300da7f17..16efa3ebb391 100644 --- a/packages/google-cloud-asset/noxfile.py +++ b/packages/google-cloud-asset/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-assured-workloads/noxfile.py b/packages/google-cloud-assured-workloads/noxfile.py index f9dd911ac1ea..179c336fc57b 100644 --- a/packages/google-cloud-assured-workloads/noxfile.py +++ b/packages/google-cloud-assured-workloads/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-audit-log/noxfile.py b/packages/google-cloud-audit-log/noxfile.py index 4724255759ff..3363deafcb99 100644 --- a/packages/google-cloud-audit-log/noxfile.py +++ b/packages/google-cloud-audit-log/noxfile.py @@ -457,37 +457,67 @@ def prerelease_deps(session, protobuf_implementation): session.install(*constraints_deps) prerel_deps = [ - "protobuf", - # dependency of grpc - "six", - "grpc-google-iam-v1", "googleapis-common-protos", - "grpcio", - "grpcio-status", "google-api-core", "google-auth", + "grpc-google-iam-v1", + "grpcio>=1.75.1" if session.python >= "3.12" else "grpcio<=1.62.2", + "grpcio-status", + "protobuf", "proto-plus", - "google-cloud-testutils", - # dependencies of google-cloud-testutils" - "click", - ] - - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--upgrade", dep) - - # Remaining dependencies - other_deps = [ - "requests", - "typing-extensions", ] - session.install(*other_deps) - # Print out prerelease package versions - session.run( - "python", "-c", "import google.protobuf; print(google.protobuf.__version__)" - ) - session.run("python", "-c", "import grpc; print(grpc.__version__)") - session.run("python", "-c", "import google.auth; print(google.auth.__version__)") + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): + print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + + if version_namespace: + session.run( + "python", + "-c", + f"import {version_namespace}; print({version_namespace}.__version__)", + ) session.run( "py.test", @@ -546,15 +576,24 @@ def core_deps_from_source(session, protobuf_implementation): session.install(*constraints_deps) core_dependencies_from_source = [ - f"{CURRENT_DIRECTORY}/../googleapis-common-protos", - f"{CURRENT_DIRECTORY}/../google-api-core", - f"{CURRENT_DIRECTORY}/../google-auth", - f"{CURRENT_DIRECTORY}/../grpc-google-iam-v1", - f"{CURRENT_DIRECTORY}/../proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--ignore-installed", "--no-deps") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-auditmanager/noxfile.py b/packages/google-cloud-auditmanager/noxfile.py index dbcbe3a1c915..b6575abc2dd7 100644 --- a/packages/google-cloud-auditmanager/noxfile.py +++ b/packages/google-cloud-auditmanager/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-automl/noxfile.py b/packages/google-cloud-automl/noxfile.py index 4e126e3abe91..fe4ed0be3334 100644 --- a/packages/google-cloud-automl/noxfile.py +++ b/packages/google-cloud-automl/noxfile.py @@ -512,24 +512,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -599,16 +626,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-backupdr/noxfile.py b/packages/google-cloud-backupdr/noxfile.py index c5b39824cd46..79518dfe63ce 100644 --- a/packages/google-cloud-backupdr/noxfile.py +++ b/packages/google-cloud-backupdr/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-bare-metal-solution/noxfile.py b/packages/google-cloud-bare-metal-solution/noxfile.py index 602905ef2bc2..ba636448dc54 100644 --- a/packages/google-cloud-bare-metal-solution/noxfile.py +++ b/packages/google-cloud-bare-metal-solution/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-batch/noxfile.py b/packages/google-cloud-batch/noxfile.py index fbe5e84b4657..50945a4bd333 100644 --- a/packages/google-cloud-batch/noxfile.py +++ b/packages/google-cloud-batch/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-beyondcorp-appconnections/noxfile.py b/packages/google-cloud-beyondcorp-appconnections/noxfile.py index 0e7e5192566b..5019a32963be 100644 --- a/packages/google-cloud-beyondcorp-appconnections/noxfile.py +++ b/packages/google-cloud-beyondcorp-appconnections/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-beyondcorp-appconnectors/noxfile.py b/packages/google-cloud-beyondcorp-appconnectors/noxfile.py index 5c50db09fb2c..a3072d33ffa0 100644 --- a/packages/google-cloud-beyondcorp-appconnectors/noxfile.py +++ b/packages/google-cloud-beyondcorp-appconnectors/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-beyondcorp-appgateways/noxfile.py b/packages/google-cloud-beyondcorp-appgateways/noxfile.py index 32407e4b5148..e0006a8fb3ec 100644 --- a/packages/google-cloud-beyondcorp-appgateways/noxfile.py +++ b/packages/google-cloud-beyondcorp-appgateways/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-beyondcorp-clientconnectorservices/noxfile.py b/packages/google-cloud-beyondcorp-clientconnectorservices/noxfile.py index 96b9aba53003..91224f861653 100644 --- a/packages/google-cloud-beyondcorp-clientconnectorservices/noxfile.py +++ b/packages/google-cloud-beyondcorp-clientconnectorservices/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-beyondcorp-clientgateways/noxfile.py b/packages/google-cloud-beyondcorp-clientgateways/noxfile.py index 970fb9406fa2..892ba5cb8e9e 100644 --- a/packages/google-cloud-beyondcorp-clientgateways/noxfile.py +++ b/packages/google-cloud-beyondcorp-clientgateways/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-biglake-hive/noxfile.py b/packages/google-cloud-biglake-hive/noxfile.py index 2c1244dce84b..6e88552f3078 100644 --- a/packages/google-cloud-biglake-hive/noxfile.py +++ b/packages/google-cloud-biglake-hive/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-biglake/noxfile.py b/packages/google-cloud-biglake/noxfile.py index 81341720fe46..d0b55a67ab87 100644 --- a/packages/google-cloud-biglake/noxfile.py +++ b/packages/google-cloud-biglake/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-bigquery-analyticshub/noxfile.py b/packages/google-cloud-bigquery-analyticshub/noxfile.py index 91d95d187304..590f23cf0fb5 100644 --- a/packages/google-cloud-bigquery-analyticshub/noxfile.py +++ b/packages/google-cloud-bigquery-analyticshub/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-bigquery-biglake/noxfile.py b/packages/google-cloud-bigquery-biglake/noxfile.py index 28a93a4d84d1..48e463c4a278 100644 --- a/packages/google-cloud-bigquery-biglake/noxfile.py +++ b/packages/google-cloud-bigquery-biglake/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-bigquery-connection/noxfile.py b/packages/google-cloud-bigquery-connection/noxfile.py index 9ea7e12c6781..ad2f5281b30d 100644 --- a/packages/google-cloud-bigquery-connection/noxfile.py +++ b/packages/google-cloud-bigquery-connection/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-bigquery-data-exchange/noxfile.py b/packages/google-cloud-bigquery-data-exchange/noxfile.py index d3aff8352cc1..2d1fce7c99fb 100644 --- a/packages/google-cloud-bigquery-data-exchange/noxfile.py +++ b/packages/google-cloud-bigquery-data-exchange/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-bigquery-datapolicies/noxfile.py b/packages/google-cloud-bigquery-datapolicies/noxfile.py index 324e25fcb5bd..9b7866c7ed0b 100644 --- a/packages/google-cloud-bigquery-datapolicies/noxfile.py +++ b/packages/google-cloud-bigquery-datapolicies/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-bigquery-datatransfer/noxfile.py b/packages/google-cloud-bigquery-datatransfer/noxfile.py index a6a0e1717ae2..fdf28fdc4dad 100644 --- a/packages/google-cloud-bigquery-datatransfer/noxfile.py +++ b/packages/google-cloud-bigquery-datatransfer/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-bigquery-logging/noxfile.py b/packages/google-cloud-bigquery-logging/noxfile.py index 560f1ca9e4fb..b8ce39acd1c8 100644 --- a/packages/google-cloud-bigquery-logging/noxfile.py +++ b/packages/google-cloud-bigquery-logging/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-bigquery-migration/noxfile.py b/packages/google-cloud-bigquery-migration/noxfile.py index 6adac136acf5..0762582c42ee 100644 --- a/packages/google-cloud-bigquery-migration/noxfile.py +++ b/packages/google-cloud-bigquery-migration/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-bigquery-reservation/noxfile.py b/packages/google-cloud-bigquery-reservation/noxfile.py index e30f8927e1b3..c3d567bdec74 100644 --- a/packages/google-cloud-bigquery-reservation/noxfile.py +++ b/packages/google-cloud-bigquery-reservation/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-bigquery-storage/noxfile.py b/packages/google-cloud-bigquery-storage/noxfile.py index 6a20073da13d..d5f8b2c331ed 100644 --- a/packages/google-cloud-bigquery-storage/noxfile.py +++ b/packages/google-cloud-bigquery-storage/noxfile.py @@ -552,24 +552,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -639,16 +666,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-bigtable/noxfile.py b/packages/google-cloud-bigtable/noxfile.py index 2a1c8d05a876..6603c4c2165c 100644 --- a/packages/google-cloud-bigtable/noxfile.py +++ b/packages/google-cloud-bigtable/noxfile.py @@ -512,37 +512,67 @@ def prerelease_deps(session, protobuf_implementation): session.install(*constraints_deps) prerel_deps = [ - "protobuf", - # dependency of grpc - "six", - "grpc-google-iam-v1", "googleapis-common-protos", - "grpcio", - "grpcio-status", "google-api-core", "google-auth", + "grpc-google-iam-v1", + "grpcio>=1.75.1" if session.python >= "3.12" else "grpcio<=1.62.2", + "grpcio-status", + "protobuf", "proto-plus", - "google-cloud-testutils", - # dependencies of google-cloud-testutils" - "click", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--upgrade", dep) + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent - # Remaining dependencies - other_deps = [ - "requests", - "cryptography", - ] - session.install(*other_deps) + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } - # Print out prerelease package versions - session.run( - "python", "-c", "import google.protobuf; print(google.protobuf.__version__)" - ) - session.run("python", "-c", "import grpc; print(grpc.__version__)") - session.run("python", "-c", "import google.auth; print(google.auth.__version__)") + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): + print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + + if version_namespace: + session.run( + "python", + "-c", + f"import {version_namespace}; print({version_namespace}.__version__)", + ) session.run( "py.test", @@ -615,16 +645,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-billing-budgets/noxfile.py b/packages/google-cloud-billing-budgets/noxfile.py index 6a040e632000..a538195651c8 100644 --- a/packages/google-cloud-billing-budgets/noxfile.py +++ b/packages/google-cloud-billing-budgets/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-billing/noxfile.py b/packages/google-cloud-billing/noxfile.py index ee4c1ca3919e..5ff16e4bce24 100644 --- a/packages/google-cloud-billing/noxfile.py +++ b/packages/google-cloud-billing/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-binary-authorization/noxfile.py b/packages/google-cloud-binary-authorization/noxfile.py index 2a4fa77cac9c..645bbf15b84a 100644 --- a/packages/google-cloud-binary-authorization/noxfile.py +++ b/packages/google-cloud-binary-authorization/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-build/noxfile.py b/packages/google-cloud-build/noxfile.py index 6270d5929536..835936fad5fe 100644 --- a/packages/google-cloud-build/noxfile.py +++ b/packages/google-cloud-build/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-capacityplanner/noxfile.py b/packages/google-cloud-capacityplanner/noxfile.py index 973ada676387..6f42baa38cb9 100644 --- a/packages/google-cloud-capacityplanner/noxfile.py +++ b/packages/google-cloud-capacityplanner/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-certificate-manager/noxfile.py b/packages/google-cloud-certificate-manager/noxfile.py index a79fdd540d62..4c57fd7aaba5 100644 --- a/packages/google-cloud-certificate-manager/noxfile.py +++ b/packages/google-cloud-certificate-manager/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-ces/noxfile.py b/packages/google-cloud-ces/noxfile.py index 79aabbb7b9ac..b7439f904429 100644 --- a/packages/google-cloud-ces/noxfile.py +++ b/packages/google-cloud-ces/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-channel/noxfile.py b/packages/google-cloud-channel/noxfile.py index 0858f2637345..d486c3adb970 100644 --- a/packages/google-cloud-channel/noxfile.py +++ b/packages/google-cloud-channel/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-chronicle/noxfile.py b/packages/google-cloud-chronicle/noxfile.py index f453d24353d7..82be42180fbd 100644 --- a/packages/google-cloud-chronicle/noxfile.py +++ b/packages/google-cloud-chronicle/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-cloudcontrolspartner/noxfile.py b/packages/google-cloud-cloudcontrolspartner/noxfile.py index ba4ab9e56d3b..aa971fab59eb 100644 --- a/packages/google-cloud-cloudcontrolspartner/noxfile.py +++ b/packages/google-cloud-cloudcontrolspartner/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-cloudsecuritycompliance/noxfile.py b/packages/google-cloud-cloudsecuritycompliance/noxfile.py index 4c054a73531f..f81441775f7d 100644 --- a/packages/google-cloud-cloudsecuritycompliance/noxfile.py +++ b/packages/google-cloud-cloudsecuritycompliance/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-commerce-consumer-procurement/noxfile.py b/packages/google-cloud-commerce-consumer-procurement/noxfile.py index 7272072eab77..e0d5b1c0cb07 100644 --- a/packages/google-cloud-commerce-consumer-procurement/noxfile.py +++ b/packages/google-cloud-commerce-consumer-procurement/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-common/noxfile.py b/packages/google-cloud-common/noxfile.py index b09750e97915..ed76a6cd34fc 100644 --- a/packages/google-cloud-common/noxfile.py +++ b/packages/google-cloud-common/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-compute-v1beta/noxfile.py b/packages/google-cloud-compute-v1beta/noxfile.py index 13260d736c46..1900897dab1b 100644 --- a/packages/google-cloud-compute-v1beta/noxfile.py +++ b/packages/google-cloud-compute-v1beta/noxfile.py @@ -510,24 +510,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -597,16 +624,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-compute/noxfile.py b/packages/google-cloud-compute/noxfile.py index 22fdcd39685e..44c8fafd23ca 100644 --- a/packages/google-cloud-compute/noxfile.py +++ b/packages/google-cloud-compute/noxfile.py @@ -510,24 +510,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -597,16 +624,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-confidentialcomputing/noxfile.py b/packages/google-cloud-confidentialcomputing/noxfile.py index 18061b330cdd..e05fddbb533d 100644 --- a/packages/google-cloud-confidentialcomputing/noxfile.py +++ b/packages/google-cloud-confidentialcomputing/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-config/noxfile.py b/packages/google-cloud-config/noxfile.py index adbe896246a7..7dec59d4965a 100644 --- a/packages/google-cloud-config/noxfile.py +++ b/packages/google-cloud-config/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-configdelivery/noxfile.py b/packages/google-cloud-configdelivery/noxfile.py index cb52dae5a5ca..ab40dec5b41a 100644 --- a/packages/google-cloud-configdelivery/noxfile.py +++ b/packages/google-cloud-configdelivery/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-contact-center-insights/noxfile.py b/packages/google-cloud-contact-center-insights/noxfile.py index af70dee50adb..1af02e79797f 100644 --- a/packages/google-cloud-contact-center-insights/noxfile.py +++ b/packages/google-cloud-contact-center-insights/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-container/noxfile.py b/packages/google-cloud-container/noxfile.py index 1706e60dbf24..258daef7fe29 100644 --- a/packages/google-cloud-container/noxfile.py +++ b/packages/google-cloud-container/noxfile.py @@ -510,24 +510,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -597,16 +624,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-containeranalysis/noxfile.py b/packages/google-cloud-containeranalysis/noxfile.py index 8d130a6479df..7795ceb16794 100644 --- a/packages/google-cloud-containeranalysis/noxfile.py +++ b/packages/google-cloud-containeranalysis/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-contentwarehouse/noxfile.py b/packages/google-cloud-contentwarehouse/noxfile.py index d2b5a6180665..f180350f481c 100644 --- a/packages/google-cloud-contentwarehouse/noxfile.py +++ b/packages/google-cloud-contentwarehouse/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-core/noxfile.py b/packages/google-cloud-core/noxfile.py index 2de40dfb9234..bcaffde3461a 100644 --- a/packages/google-cloud-core/noxfile.py +++ b/packages/google-cloud-core/noxfile.py @@ -288,13 +288,64 @@ def prerelease_deps(session, protobuf_implementation): "googleapis-common-protos", "google-api-core", "google-auth", - "grpcio", + "grpc-google-iam-v1", + "grpcio>=1.75.1" if session.python >= "3.12" else "grpcio<=1.62.2", "grpcio-status", + "protobuf", "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--upgrade", dep) + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): + print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + + if version_namespace: + session.run( + "python", + "-c", + f"import {version_namespace}; print({version_namespace}.__version__)", + ) session.run( "py.test", @@ -368,21 +419,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - f"{CURRENT_DIRECTORY}/../googleapis-common-protos", - f"{CURRENT_DIRECTORY}/../google-api-core", - f"{CURRENT_DIRECTORY}/../google-auth", - f"{CURRENT_DIRECTORY}/../grpc-google-iam-v1", - f"{CURRENT_DIRECTORY}/../proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] - for dep in core_dependencies_from_source: - session.install("--pre", "--no-deps", "--upgrade", dep) + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent - # Remaining dependencies - other_deps = [ - "cachetools", - ] - session.install(*other_deps) + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + + for dep in core_dependencies_from_source: + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-data-fusion/noxfile.py b/packages/google-cloud-data-fusion/noxfile.py index 2e3156a449df..05f45779c034 100644 --- a/packages/google-cloud-data-fusion/noxfile.py +++ b/packages/google-cloud-data-fusion/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-data-qna/noxfile.py b/packages/google-cloud-data-qna/noxfile.py index f953fc9b7d18..e33677f53ca4 100644 --- a/packages/google-cloud-data-qna/noxfile.py +++ b/packages/google-cloud-data-qna/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-databasecenter/noxfile.py b/packages/google-cloud-databasecenter/noxfile.py index 9afd87e584e5..d511829b2c26 100644 --- a/packages/google-cloud-databasecenter/noxfile.py +++ b/packages/google-cloud-databasecenter/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-datacatalog-lineage-configmanagement/noxfile.py b/packages/google-cloud-datacatalog-lineage-configmanagement/noxfile.py index 263b897c4d05..41ee6fa0792c 100644 --- a/packages/google-cloud-datacatalog-lineage-configmanagement/noxfile.py +++ b/packages/google-cloud-datacatalog-lineage-configmanagement/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-datacatalog-lineage/noxfile.py b/packages/google-cloud-datacatalog-lineage/noxfile.py index af54a9b59e51..95df85293533 100644 --- a/packages/google-cloud-datacatalog-lineage/noxfile.py +++ b/packages/google-cloud-datacatalog-lineage/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-datacatalog/noxfile.py b/packages/google-cloud-datacatalog/noxfile.py index 3eee55c531d3..389e487f05c6 100644 --- a/packages/google-cloud-datacatalog/noxfile.py +++ b/packages/google-cloud-datacatalog/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-dataflow-client/noxfile.py b/packages/google-cloud-dataflow-client/noxfile.py index 8ac1db8f7c77..779dac8ce9d6 100644 --- a/packages/google-cloud-dataflow-client/noxfile.py +++ b/packages/google-cloud-dataflow-client/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-dataform/noxfile.py b/packages/google-cloud-dataform/noxfile.py index 623552014adc..45613382879c 100644 --- a/packages/google-cloud-dataform/noxfile.py +++ b/packages/google-cloud-dataform/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-datalabeling/noxfile.py b/packages/google-cloud-datalabeling/noxfile.py index b0a11cebd423..2cd4fdd62cf6 100644 --- a/packages/google-cloud-datalabeling/noxfile.py +++ b/packages/google-cloud-datalabeling/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-dataplex/noxfile.py b/packages/google-cloud-dataplex/noxfile.py index a1bbf60d99bd..d55a18d9800b 100644 --- a/packages/google-cloud-dataplex/noxfile.py +++ b/packages/google-cloud-dataplex/noxfile.py @@ -510,24 +510,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -597,16 +624,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-dataproc-metastore/noxfile.py b/packages/google-cloud-dataproc-metastore/noxfile.py index 192b66fdb052..e5eac35ede0b 100644 --- a/packages/google-cloud-dataproc-metastore/noxfile.py +++ b/packages/google-cloud-dataproc-metastore/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-dataproc/noxfile.py b/packages/google-cloud-dataproc/noxfile.py index 674c7c6e570c..b877b623bfe4 100644 --- a/packages/google-cloud-dataproc/noxfile.py +++ b/packages/google-cloud-dataproc/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-datastore/noxfile.py b/packages/google-cloud-datastore/noxfile.py index 7f1e24e8628c..c198e63bfd5f 100644 --- a/packages/google-cloud-datastore/noxfile.py +++ b/packages/google-cloud-datastore/noxfile.py @@ -519,24 +519,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -606,16 +633,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-datastream/noxfile.py b/packages/google-cloud-datastream/noxfile.py index cad9d2ffc7ad..2da0a49186c7 100644 --- a/packages/google-cloud-datastream/noxfile.py +++ b/packages/google-cloud-datastream/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-deploy/noxfile.py b/packages/google-cloud-deploy/noxfile.py index 7dbe97a24043..9092c268f46a 100644 --- a/packages/google-cloud-deploy/noxfile.py +++ b/packages/google-cloud-deploy/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-developerconnect/noxfile.py b/packages/google-cloud-developerconnect/noxfile.py index feea190d1046..e6ff0354bdcf 100644 --- a/packages/google-cloud-developerconnect/noxfile.py +++ b/packages/google-cloud-developerconnect/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-devicestreaming/noxfile.py b/packages/google-cloud-devicestreaming/noxfile.py index a36616932b10..8637ea58f230 100644 --- a/packages/google-cloud-devicestreaming/noxfile.py +++ b/packages/google-cloud-devicestreaming/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-dialogflow-cx/noxfile.py b/packages/google-cloud-dialogflow-cx/noxfile.py index 25d4b5fca8e2..002db80062e3 100644 --- a/packages/google-cloud-dialogflow-cx/noxfile.py +++ b/packages/google-cloud-dialogflow-cx/noxfile.py @@ -510,24 +510,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -597,16 +624,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-dialogflow/noxfile.py b/packages/google-cloud-dialogflow/noxfile.py index 33e358ccffb1..d898b5a2e233 100644 --- a/packages/google-cloud-dialogflow/noxfile.py +++ b/packages/google-cloud-dialogflow/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-discoveryengine/noxfile.py b/packages/google-cloud-discoveryengine/noxfile.py index 6f75c2183cee..4f7e6acea172 100644 --- a/packages/google-cloud-discoveryengine/noxfile.py +++ b/packages/google-cloud-discoveryengine/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-dlp/noxfile.py b/packages/google-cloud-dlp/noxfile.py index 4b6d97d95954..89b764d81bec 100644 --- a/packages/google-cloud-dlp/noxfile.py +++ b/packages/google-cloud-dlp/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-dms/noxfile.py b/packages/google-cloud-dms/noxfile.py index 430a8127e1a3..5df3dc29e6d0 100644 --- a/packages/google-cloud-dms/noxfile.py +++ b/packages/google-cloud-dms/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-dns/noxfile.py b/packages/google-cloud-dns/noxfile.py index bb38d1e08a91..39a568b945be 100644 --- a/packages/google-cloud-dns/noxfile.py +++ b/packages/google-cloud-dns/noxfile.py @@ -415,35 +415,67 @@ def prerelease_deps(session): session.install(*constraints_deps) prerel_deps = [ - "protobuf", - "grpc-google-iam-v1", "googleapis-common-protos", - "grpcio", - "grpcio-status", - "google-cloud-core", + "google-api-core", "google-auth", + "grpc-google-iam-v1", + "grpcio>=1.75.1" if session.python >= "3.12" else "grpcio<=1.62.2", + "grpcio-status", + "protobuf", "proto-plus", - "google-cloud-testutils", - # dependencies of google-cloud-testutils" - "click", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--upgrade", dep) - - # Remaining dependencies - other_deps = [ - "requests", - "typing-extensions", - ] - session.install(*other_deps) + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): + print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) - # Print out prerelease package versions - session.run( - "python", "-c", "import google.protobuf; print(google.protobuf.__version__)" - ) - session.run("python", "-c", "import grpc; print(grpc.__version__)") - session.run("python", "-c", "import google.auth; print(google.auth.__version__)") + if version_namespace: + session.run( + "python", + "-c", + f"import {version_namespace}; print({version_namespace}.__version__)", + ) session.run( "py.test", @@ -487,27 +519,24 @@ def core_deps_from_source(session): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - f"{CURRENT_DIRECTORY}/../googleapis-common-protos", - f"{CURRENT_DIRECTORY}/../google-api-core", - f"{CURRENT_DIRECTORY}/../google-auth", - f"{CURRENT_DIRECTORY}/../grpc-google-iam-v1", - f"{CURRENT_DIRECTORY}/../proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] - for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") - # Remaining dependencies - other_deps = [ - "requests", - ] - session.install(*other_deps) + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent - # Print out prerelease package versions - session.run( - "python", "-c", "import google.protobuf; print(google.protobuf.__version__)" - ) - session.run("python", "-c", "import google.auth; print(google.auth.__version__)") + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + + for dep in core_dependencies_from_source: + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-documentai-toolbox/noxfile.py b/packages/google-cloud-documentai-toolbox/noxfile.py index db70716c5d6b..37a21c41ed0c 100644 --- a/packages/google-cloud-documentai-toolbox/noxfile.py +++ b/packages/google-cloud-documentai-toolbox/noxfile.py @@ -419,36 +419,67 @@ def prerelease_deps(session, protobuf_implementation): session.install(*constraints_deps) prerel_deps = [ - "protobuf", - # dependency of grpc - "six", - "grpc-google-iam-v1", "googleapis-common-protos", - "grpcio", - "grpcio-status", "google-api-core", "google-auth", + "grpc-google-iam-v1", + "grpcio>=1.75.1" if session.python >= "3.12" else "grpcio<=1.62.2", + "grpcio-status", + "protobuf", "proto-plus", - "google-cloud-testutils", - # dependencies of google-cloud-testutils" - "click", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--upgrade", dep) - - # Remaining dependencies - other_deps = [ - "requests", - ] - session.install(*other_deps) + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): + print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) - # Print out prerelease package versions - session.run( - "python", "-c", "import google.protobuf; print(google.protobuf.__version__)" - ) - session.run("python", "-c", "import grpc; print(grpc.__version__)") - session.run("python", "-c", "import google.auth; print(google.auth.__version__)") + if version_namespace: + session.run( + "python", + "-c", + f"import {version_namespace}; print({version_namespace}.__version__)", + ) session.run( "py.test", @@ -519,18 +550,24 @@ def core_deps_from_source(session, protobuf_implementation): install_unittest_dependencies(session, "-c", constraints_path) core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] - for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent - tests_path = os.path.join("tests", "unit") + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + + for dep in core_dependencies_from_source: + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", "--quiet", diff --git a/packages/google-cloud-documentai/noxfile.py b/packages/google-cloud-documentai/noxfile.py index 63c4a74fcf07..809b76bb2059 100644 --- a/packages/google-cloud-documentai/noxfile.py +++ b/packages/google-cloud-documentai/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-domains/noxfile.py b/packages/google-cloud-domains/noxfile.py index 62c75593db3e..fd379f095b6f 100644 --- a/packages/google-cloud-domains/noxfile.py +++ b/packages/google-cloud-domains/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-edgecontainer/noxfile.py b/packages/google-cloud-edgecontainer/noxfile.py index 74d5b46b07c3..881f49d2738c 100644 --- a/packages/google-cloud-edgecontainer/noxfile.py +++ b/packages/google-cloud-edgecontainer/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-edgenetwork/noxfile.py b/packages/google-cloud-edgenetwork/noxfile.py index 64fcbb19b148..3aa46b03735d 100644 --- a/packages/google-cloud-edgenetwork/noxfile.py +++ b/packages/google-cloud-edgenetwork/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-enterpriseknowledgegraph/noxfile.py b/packages/google-cloud-enterpriseknowledgegraph/noxfile.py index 753c70a55c41..b0ebc1d8d3e1 100644 --- a/packages/google-cloud-enterpriseknowledgegraph/noxfile.py +++ b/packages/google-cloud-enterpriseknowledgegraph/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-error-reporting/noxfile.py b/packages/google-cloud-error-reporting/noxfile.py index 1f4fd2f44b39..8752e1c2b9ca 100644 --- a/packages/google-cloud-error-reporting/noxfile.py +++ b/packages/google-cloud-error-reporting/noxfile.py @@ -512,24 +512,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -599,16 +626,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-essential-contacts/noxfile.py b/packages/google-cloud-essential-contacts/noxfile.py index caa82cdb2bb3..b4ec00a9b963 100644 --- a/packages/google-cloud-essential-contacts/noxfile.py +++ b/packages/google-cloud-essential-contacts/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-eventarc-publishing/noxfile.py b/packages/google-cloud-eventarc-publishing/noxfile.py index d1b3e6027eee..00430591c30f 100644 --- a/packages/google-cloud-eventarc-publishing/noxfile.py +++ b/packages/google-cloud-eventarc-publishing/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-eventarc/noxfile.py b/packages/google-cloud-eventarc/noxfile.py index 0a423fa2f88e..81fb2d09b33a 100644 --- a/packages/google-cloud-eventarc/noxfile.py +++ b/packages/google-cloud-eventarc/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-filestore/noxfile.py b/packages/google-cloud-filestore/noxfile.py index 3f3181edcbc1..cd5eba850869 100644 --- a/packages/google-cloud-filestore/noxfile.py +++ b/packages/google-cloud-filestore/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-financialservices/noxfile.py b/packages/google-cloud-financialservices/noxfile.py index 3bc3142cd191..62006460c8f6 100644 --- a/packages/google-cloud-financialservices/noxfile.py +++ b/packages/google-cloud-financialservices/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-firestore/noxfile.py b/packages/google-cloud-firestore/noxfile.py index 588dd7c0058d..cf042b78ac60 100644 --- a/packages/google-cloud-firestore/noxfile.py +++ b/packages/google-cloud-firestore/noxfile.py @@ -570,24 +570,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -657,16 +684,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-functions/noxfile.py b/packages/google-cloud-functions/noxfile.py index 9697516fdfe6..511cba8fb93b 100644 --- a/packages/google-cloud-functions/noxfile.py +++ b/packages/google-cloud-functions/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-gdchardwaremanagement/noxfile.py b/packages/google-cloud-gdchardwaremanagement/noxfile.py index 13da357e3c79..06795c0c8718 100644 --- a/packages/google-cloud-gdchardwaremanagement/noxfile.py +++ b/packages/google-cloud-gdchardwaremanagement/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-geminidataanalytics/noxfile.py b/packages/google-cloud-geminidataanalytics/noxfile.py index 21d70d84fe77..487f18e3f6d7 100644 --- a/packages/google-cloud-geminidataanalytics/noxfile.py +++ b/packages/google-cloud-geminidataanalytics/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-gke-backup/noxfile.py b/packages/google-cloud-gke-backup/noxfile.py index daf624fbc5d3..83dc3f0ab28b 100644 --- a/packages/google-cloud-gke-backup/noxfile.py +++ b/packages/google-cloud-gke-backup/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-gke-connect-gateway/noxfile.py b/packages/google-cloud-gke-connect-gateway/noxfile.py index b8df14b0bf30..3952dc406a34 100644 --- a/packages/google-cloud-gke-connect-gateway/noxfile.py +++ b/packages/google-cloud-gke-connect-gateway/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-gke-hub/noxfile.py b/packages/google-cloud-gke-hub/noxfile.py index cbaad70c79b9..2716ad8ed3f8 100644 --- a/packages/google-cloud-gke-hub/noxfile.py +++ b/packages/google-cloud-gke-hub/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-gke-multicloud/noxfile.py b/packages/google-cloud-gke-multicloud/noxfile.py index 7c6ae85a027b..074da3727afb 100644 --- a/packages/google-cloud-gke-multicloud/noxfile.py +++ b/packages/google-cloud-gke-multicloud/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-gkerecommender/noxfile.py b/packages/google-cloud-gkerecommender/noxfile.py index 71f24501c176..9b1fcf9b264f 100644 --- a/packages/google-cloud-gkerecommender/noxfile.py +++ b/packages/google-cloud-gkerecommender/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-gsuiteaddons/noxfile.py b/packages/google-cloud-gsuiteaddons/noxfile.py index 30efc29dd192..9970e4961d7e 100644 --- a/packages/google-cloud-gsuiteaddons/noxfile.py +++ b/packages/google-cloud-gsuiteaddons/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-hypercomputecluster/noxfile.py b/packages/google-cloud-hypercomputecluster/noxfile.py index 5a05ab3c52ab..bb435cc3de5d 100644 --- a/packages/google-cloud-hypercomputecluster/noxfile.py +++ b/packages/google-cloud-hypercomputecluster/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-iam-logging/noxfile.py b/packages/google-cloud-iam-logging/noxfile.py index 396f3fb8f2e0..cbcb47571ad0 100644 --- a/packages/google-cloud-iam-logging/noxfile.py +++ b/packages/google-cloud-iam-logging/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-iam/noxfile.py b/packages/google-cloud-iam/noxfile.py index 685d0000406f..5efca12d3882 100644 --- a/packages/google-cloud-iam/noxfile.py +++ b/packages/google-cloud-iam/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-iamconnectorcredentials/noxfile.py b/packages/google-cloud-iamconnectorcredentials/noxfile.py index 1a62b2bece5c..904e5b509143 100644 --- a/packages/google-cloud-iamconnectorcredentials/noxfile.py +++ b/packages/google-cloud-iamconnectorcredentials/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-iap/noxfile.py b/packages/google-cloud-iap/noxfile.py index 3d9e10f804cc..262c4a840a8b 100644 --- a/packages/google-cloud-iap/noxfile.py +++ b/packages/google-cloud-iap/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-ids/noxfile.py b/packages/google-cloud-ids/noxfile.py index d4e71f7d9f6c..c8e556a57019 100644 --- a/packages/google-cloud-ids/noxfile.py +++ b/packages/google-cloud-ids/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-kms-inventory/noxfile.py b/packages/google-cloud-kms-inventory/noxfile.py index ba099f2f62c0..95f05359c7e7 100644 --- a/packages/google-cloud-kms-inventory/noxfile.py +++ b/packages/google-cloud-kms-inventory/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-kms/noxfile.py b/packages/google-cloud-kms/noxfile.py index 184c52d7bf3b..e138ba8ff28b 100644 --- a/packages/google-cloud-kms/noxfile.py +++ b/packages/google-cloud-kms/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-language/noxfile.py b/packages/google-cloud-language/noxfile.py index 9589920a56e4..5472c1d3ec35 100644 --- a/packages/google-cloud-language/noxfile.py +++ b/packages/google-cloud-language/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-licensemanager/noxfile.py b/packages/google-cloud-licensemanager/noxfile.py index ad8ab48d566a..a94eedcedf92 100644 --- a/packages/google-cloud-licensemanager/noxfile.py +++ b/packages/google-cloud-licensemanager/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-life-sciences/noxfile.py b/packages/google-cloud-life-sciences/noxfile.py index 3e621ecfc948..0e29b7608f26 100644 --- a/packages/google-cloud-life-sciences/noxfile.py +++ b/packages/google-cloud-life-sciences/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-locationfinder/noxfile.py b/packages/google-cloud-locationfinder/noxfile.py index 4d89fa733a9f..eb16353af865 100644 --- a/packages/google-cloud-locationfinder/noxfile.py +++ b/packages/google-cloud-locationfinder/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-logging/noxfile.py b/packages/google-cloud-logging/noxfile.py index fe328913d477..c9072adcd251 100644 --- a/packages/google-cloud-logging/noxfile.py +++ b/packages/google-cloud-logging/noxfile.py @@ -510,24 +510,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -597,16 +624,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-lustre/noxfile.py b/packages/google-cloud-lustre/noxfile.py index 53c7f39c9653..c440e4dddd87 100644 --- a/packages/google-cloud-lustre/noxfile.py +++ b/packages/google-cloud-lustre/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-maintenance-api/noxfile.py b/packages/google-cloud-maintenance-api/noxfile.py index a72c0f17021a..6d605055ac57 100644 --- a/packages/google-cloud-maintenance-api/noxfile.py +++ b/packages/google-cloud-maintenance-api/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-managed-identities/noxfile.py b/packages/google-cloud-managed-identities/noxfile.py index e2f9b8fa2727..4ade223dc79c 100644 --- a/packages/google-cloud-managed-identities/noxfile.py +++ b/packages/google-cloud-managed-identities/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-managedkafka-schemaregistry/noxfile.py b/packages/google-cloud-managedkafka-schemaregistry/noxfile.py index 622a6fa8adb4..60b8ba1c499e 100644 --- a/packages/google-cloud-managedkafka-schemaregistry/noxfile.py +++ b/packages/google-cloud-managedkafka-schemaregistry/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-managedkafka/noxfile.py b/packages/google-cloud-managedkafka/noxfile.py index cf9f1a957cb5..f5d88172ea5f 100644 --- a/packages/google-cloud-managedkafka/noxfile.py +++ b/packages/google-cloud-managedkafka/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-media-translation/noxfile.py b/packages/google-cloud-media-translation/noxfile.py index 5c0518e99ffe..8522262161a3 100644 --- a/packages/google-cloud-media-translation/noxfile.py +++ b/packages/google-cloud-media-translation/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-memcache/noxfile.py b/packages/google-cloud-memcache/noxfile.py index c0ef1fb78a0c..475505c61e6e 100644 --- a/packages/google-cloud-memcache/noxfile.py +++ b/packages/google-cloud-memcache/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-memorystore/noxfile.py b/packages/google-cloud-memorystore/noxfile.py index 0815f92794a5..aabc0b0590c4 100644 --- a/packages/google-cloud-memorystore/noxfile.py +++ b/packages/google-cloud-memorystore/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-migrationcenter/noxfile.py b/packages/google-cloud-migrationcenter/noxfile.py index 5be4c588c05d..ad5e0e6fb38b 100644 --- a/packages/google-cloud-migrationcenter/noxfile.py +++ b/packages/google-cloud-migrationcenter/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-modelarmor/noxfile.py b/packages/google-cloud-modelarmor/noxfile.py index f2596eef3931..e5c7b4e9f16c 100644 --- a/packages/google-cloud-modelarmor/noxfile.py +++ b/packages/google-cloud-modelarmor/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-monitoring-dashboards/noxfile.py b/packages/google-cloud-monitoring-dashboards/noxfile.py index 0a94fad7ba87..14ff67567faf 100644 --- a/packages/google-cloud-monitoring-dashboards/noxfile.py +++ b/packages/google-cloud-monitoring-dashboards/noxfile.py @@ -512,24 +512,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -599,16 +626,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-monitoring-metrics-scopes/noxfile.py b/packages/google-cloud-monitoring-metrics-scopes/noxfile.py index 0eae4d44ed46..0612f3c5c9d5 100644 --- a/packages/google-cloud-monitoring-metrics-scopes/noxfile.py +++ b/packages/google-cloud-monitoring-metrics-scopes/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-monitoring/noxfile.py b/packages/google-cloud-monitoring/noxfile.py index 776c306158d6..89809b11f6a9 100644 --- a/packages/google-cloud-monitoring/noxfile.py +++ b/packages/google-cloud-monitoring/noxfile.py @@ -513,24 +513,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -600,16 +627,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-ndb/noxfile.py b/packages/google-cloud-ndb/noxfile.py index 9234e413582a..9899bac7d869 100644 --- a/packages/google-cloud-ndb/noxfile.py +++ b/packages/google-cloud-ndb/noxfile.py @@ -128,37 +128,67 @@ def prerelease_deps(session, protobuf_implementation): session.install(*constraints_deps) prerel_deps = [ - "protobuf", - # dependency of grpc - "six", - "grpc-google-iam-v1", - "google-cloud-datastore", "googleapis-common-protos", - "grpcio", - "grpcio-status", "google-api-core", "google-auth", + "grpc-google-iam-v1", + "grpcio>=1.75.1" if session.python >= "3.12" else "grpcio<=1.62.2", + "grpcio-status", + "protobuf", "proto-plus", - "google-cloud-testutils", - # dependencies of google-cloud-testutils" - "click", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--upgrade", dep) + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent - # Remaining dependencies - other_deps = [ - "requests", - ] - session.install(*other_deps) + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } - # Print out prerelease package versions - session.run( - "python", "-c", "import google.protobuf; print(google.protobuf.__version__)" - ) - session.run("python", "-c", "import grpc; print(grpc.__version__)") - session.run("python", "-c", "import google.auth; print(google.auth.__version__)") + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): + print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + + if version_namespace: + session.run( + "python", + "-c", + f"import {version_namespace}; print({version_namespace}.__version__)", + ) session.run( "py.test", @@ -455,18 +485,24 @@ def core_deps_from_source(session, protobuf_implementation): install_unittest_dependencies(session, "-c", constraints_path) core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] - for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent - tests_path = os.path.join("tests", "unit") + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + + for dep in core_dependencies_from_source: + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", "--quiet", diff --git a/packages/google-cloud-netapp/noxfile.py b/packages/google-cloud-netapp/noxfile.py index ce2f63b896a2..63e519fd51cf 100644 --- a/packages/google-cloud-netapp/noxfile.py +++ b/packages/google-cloud-netapp/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-network-connectivity/noxfile.py b/packages/google-cloud-network-connectivity/noxfile.py index 5e594c944616..ab9cdffc3d09 100644 --- a/packages/google-cloud-network-connectivity/noxfile.py +++ b/packages/google-cloud-network-connectivity/noxfile.py @@ -510,24 +510,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -597,16 +624,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-network-management/noxfile.py b/packages/google-cloud-network-management/noxfile.py index 028d77ce472e..6ab81ee2a3c4 100644 --- a/packages/google-cloud-network-management/noxfile.py +++ b/packages/google-cloud-network-management/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-network-security/noxfile.py b/packages/google-cloud-network-security/noxfile.py index c0cd64c4a441..91987127cad4 100644 --- a/packages/google-cloud-network-security/noxfile.py +++ b/packages/google-cloud-network-security/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-network-services/noxfile.py b/packages/google-cloud-network-services/noxfile.py index ed5d8dfe9b33..f673950df530 100644 --- a/packages/google-cloud-network-services/noxfile.py +++ b/packages/google-cloud-network-services/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-notebooks/noxfile.py b/packages/google-cloud-notebooks/noxfile.py index 7b1badad6cbb..87a0e5d0164b 100644 --- a/packages/google-cloud-notebooks/noxfile.py +++ b/packages/google-cloud-notebooks/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-optimization/noxfile.py b/packages/google-cloud-optimization/noxfile.py index ad11b9be0c15..2eaae472b110 100644 --- a/packages/google-cloud-optimization/noxfile.py +++ b/packages/google-cloud-optimization/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-oracledatabase/noxfile.py b/packages/google-cloud-oracledatabase/noxfile.py index 57ad10caa0e2..35d8648aeb1b 100644 --- a/packages/google-cloud-oracledatabase/noxfile.py +++ b/packages/google-cloud-oracledatabase/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-orchestration-airflow/noxfile.py b/packages/google-cloud-orchestration-airflow/noxfile.py index cbdfd79f5f5e..841f842cd053 100644 --- a/packages/google-cloud-orchestration-airflow/noxfile.py +++ b/packages/google-cloud-orchestration-airflow/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-org-policy/noxfile.py b/packages/google-cloud-org-policy/noxfile.py index 8afd34103100..69f771d2dad0 100644 --- a/packages/google-cloud-org-policy/noxfile.py +++ b/packages/google-cloud-org-policy/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-os-config/noxfile.py b/packages/google-cloud-os-config/noxfile.py index 6c6b02e6d5e2..aea194c90c0b 100644 --- a/packages/google-cloud-os-config/noxfile.py +++ b/packages/google-cloud-os-config/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-os-login/noxfile.py b/packages/google-cloud-os-login/noxfile.py index f42daa9efda7..8603fd283634 100644 --- a/packages/google-cloud-os-login/noxfile.py +++ b/packages/google-cloud-os-login/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-parallelstore/noxfile.py b/packages/google-cloud-parallelstore/noxfile.py index e674b4465a20..20716d296d47 100644 --- a/packages/google-cloud-parallelstore/noxfile.py +++ b/packages/google-cloud-parallelstore/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-parametermanager/noxfile.py b/packages/google-cloud-parametermanager/noxfile.py index 1c1ba6092777..b84591a664ec 100644 --- a/packages/google-cloud-parametermanager/noxfile.py +++ b/packages/google-cloud-parametermanager/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-phishing-protection/noxfile.py b/packages/google-cloud-phishing-protection/noxfile.py index 0a2f735c3382..53a2485fac25 100644 --- a/packages/google-cloud-phishing-protection/noxfile.py +++ b/packages/google-cloud-phishing-protection/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-policy-troubleshooter/noxfile.py b/packages/google-cloud-policy-troubleshooter/noxfile.py index 25baaea8f0eb..2ea2d4e97bca 100644 --- a/packages/google-cloud-policy-troubleshooter/noxfile.py +++ b/packages/google-cloud-policy-troubleshooter/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-policysimulator/noxfile.py b/packages/google-cloud-policysimulator/noxfile.py index 3104273791e9..569913901c8b 100644 --- a/packages/google-cloud-policysimulator/noxfile.py +++ b/packages/google-cloud-policysimulator/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-policytroubleshooter-iam/noxfile.py b/packages/google-cloud-policytroubleshooter-iam/noxfile.py index cfecc4c5432c..88bd172adb4a 100644 --- a/packages/google-cloud-policytroubleshooter-iam/noxfile.py +++ b/packages/google-cloud-policytroubleshooter-iam/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-private-ca/noxfile.py b/packages/google-cloud-private-ca/noxfile.py index 928d0a09d751..fbf8c36a0016 100644 --- a/packages/google-cloud-private-ca/noxfile.py +++ b/packages/google-cloud-private-ca/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-private-catalog/noxfile.py b/packages/google-cloud-private-catalog/noxfile.py index 97f12a45d4de..66dafb397418 100644 --- a/packages/google-cloud-private-catalog/noxfile.py +++ b/packages/google-cloud-private-catalog/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-privilegedaccessmanager/noxfile.py b/packages/google-cloud-privilegedaccessmanager/noxfile.py index 2e6f9a28e989..dbc9842cb20b 100644 --- a/packages/google-cloud-privilegedaccessmanager/noxfile.py +++ b/packages/google-cloud-privilegedaccessmanager/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-pubsub/noxfile.py b/packages/google-cloud-pubsub/noxfile.py index 0071c6b0f230..28e2a154c01a 100644 --- a/packages/google-cloud-pubsub/noxfile.py +++ b/packages/google-cloud-pubsub/noxfile.py @@ -560,24 +560,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", diff --git a/packages/google-cloud-quotas/noxfile.py b/packages/google-cloud-quotas/noxfile.py index d36f94afd752..6ee545ec56cc 100644 --- a/packages/google-cloud-quotas/noxfile.py +++ b/packages/google-cloud-quotas/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-rapidmigrationassessment/noxfile.py b/packages/google-cloud-rapidmigrationassessment/noxfile.py index 97194351c99c..867c29571e21 100644 --- a/packages/google-cloud-rapidmigrationassessment/noxfile.py +++ b/packages/google-cloud-rapidmigrationassessment/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-recaptcha-enterprise/noxfile.py b/packages/google-cloud-recaptcha-enterprise/noxfile.py index 9bf9e14345d7..3a2436d3d95d 100644 --- a/packages/google-cloud-recaptcha-enterprise/noxfile.py +++ b/packages/google-cloud-recaptcha-enterprise/noxfile.py @@ -510,24 +510,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -597,16 +624,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-recommendations-ai/noxfile.py b/packages/google-cloud-recommendations-ai/noxfile.py index 95ef9d61b02b..976d599c64fe 100644 --- a/packages/google-cloud-recommendations-ai/noxfile.py +++ b/packages/google-cloud-recommendations-ai/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-recommender/noxfile.py b/packages/google-cloud-recommender/noxfile.py index f2084d82cf02..ce35327c7219 100644 --- a/packages/google-cloud-recommender/noxfile.py +++ b/packages/google-cloud-recommender/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-redis-cluster/noxfile.py b/packages/google-cloud-redis-cluster/noxfile.py index 982836af0bee..c66c03a958fe 100644 --- a/packages/google-cloud-redis-cluster/noxfile.py +++ b/packages/google-cloud-redis-cluster/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-redis/noxfile.py b/packages/google-cloud-redis/noxfile.py index 2cf51925b6c6..267566c53282 100644 --- a/packages/google-cloud-redis/noxfile.py +++ b/packages/google-cloud-redis/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-resource-manager/noxfile.py b/packages/google-cloud-resource-manager/noxfile.py index 19a87b36b89f..ed875a33b500 100644 --- a/packages/google-cloud-resource-manager/noxfile.py +++ b/packages/google-cloud-resource-manager/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-retail/noxfile.py b/packages/google-cloud-retail/noxfile.py index d281dc7ac174..3e8e20f3739e 100644 --- a/packages/google-cloud-retail/noxfile.py +++ b/packages/google-cloud-retail/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-run/noxfile.py b/packages/google-cloud-run/noxfile.py index 25c87fa037d7..ba5ee65a3053 100644 --- a/packages/google-cloud-run/noxfile.py +++ b/packages/google-cloud-run/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-runtimeconfig/noxfile.py b/packages/google-cloud-runtimeconfig/noxfile.py index 42e77572607e..6c732305962c 100644 --- a/packages/google-cloud-runtimeconfig/noxfile.py +++ b/packages/google-cloud-runtimeconfig/noxfile.py @@ -407,11 +407,67 @@ def prerelease_deps(session): session.install(*constraints_deps) prerel_deps = [ - "google-cloud-core", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "grpcio>=1.75.1" if session.python >= "3.12" else "grpcio<=1.62.2", + "grpcio-status", + "protobuf", + "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--upgrade", dep) + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): + print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + + if version_namespace: + session.run( + "python", + "-c", + f"import {version_namespace}; print({version_namespace}.__version__)", + ) session.run( "py.test", @@ -478,11 +534,24 @@ def core_deps_from_source(session): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - f"{CURRENT_DIRECTORY}/../google-cloud-core", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install("--pre", "--no-deps", "--upgrade", dep) + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-saasplatform-saasservicemgmt/noxfile.py b/packages/google-cloud-saasplatform-saasservicemgmt/noxfile.py index 71390773f34a..4e0e7af4c332 100644 --- a/packages/google-cloud-saasplatform-saasservicemgmt/noxfile.py +++ b/packages/google-cloud-saasplatform-saasservicemgmt/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-scheduler/noxfile.py b/packages/google-cloud-scheduler/noxfile.py index e587bd892d8c..18566636e1e5 100644 --- a/packages/google-cloud-scheduler/noxfile.py +++ b/packages/google-cloud-scheduler/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-secret-manager/noxfile.py b/packages/google-cloud-secret-manager/noxfile.py index ed8df184305b..64f6d90685e3 100644 --- a/packages/google-cloud-secret-manager/noxfile.py +++ b/packages/google-cloud-secret-manager/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-securesourcemanager/noxfile.py b/packages/google-cloud-securesourcemanager/noxfile.py index 0e2d2a3d3565..0c7903a13a08 100644 --- a/packages/google-cloud-securesourcemanager/noxfile.py +++ b/packages/google-cloud-securesourcemanager/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-security-publicca/noxfile.py b/packages/google-cloud-security-publicca/noxfile.py index ed71015f1b17..ae30df50a418 100644 --- a/packages/google-cloud-security-publicca/noxfile.py +++ b/packages/google-cloud-security-publicca/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-securitycenter/noxfile.py b/packages/google-cloud-securitycenter/noxfile.py index e039fb382140..6992bcf712f8 100644 --- a/packages/google-cloud-securitycenter/noxfile.py +++ b/packages/google-cloud-securitycenter/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-securitycentermanagement/noxfile.py b/packages/google-cloud-securitycentermanagement/noxfile.py index e9cd2b625e22..0a6602371319 100644 --- a/packages/google-cloud-securitycentermanagement/noxfile.py +++ b/packages/google-cloud-securitycentermanagement/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-service-control/noxfile.py b/packages/google-cloud-service-control/noxfile.py index 7a1445cab712..b0ce0c6e8a17 100644 --- a/packages/google-cloud-service-control/noxfile.py +++ b/packages/google-cloud-service-control/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-service-directory/noxfile.py b/packages/google-cloud-service-directory/noxfile.py index 807b37352aa4..880032cc464a 100644 --- a/packages/google-cloud-service-directory/noxfile.py +++ b/packages/google-cloud-service-directory/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-service-management/noxfile.py b/packages/google-cloud-service-management/noxfile.py index b83934a03615..71fa92d48304 100644 --- a/packages/google-cloud-service-management/noxfile.py +++ b/packages/google-cloud-service-management/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-service-usage/noxfile.py b/packages/google-cloud-service-usage/noxfile.py index df6e234e6202..b3fcdc4c530f 100644 --- a/packages/google-cloud-service-usage/noxfile.py +++ b/packages/google-cloud-service-usage/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-servicehealth/noxfile.py b/packages/google-cloud-servicehealth/noxfile.py index a4f1115c66b9..5ad977349981 100644 --- a/packages/google-cloud-servicehealth/noxfile.py +++ b/packages/google-cloud-servicehealth/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-shell/noxfile.py b/packages/google-cloud-shell/noxfile.py index 0bb0f66d758a..104d760f50b3 100644 --- a/packages/google-cloud-shell/noxfile.py +++ b/packages/google-cloud-shell/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-source-context/noxfile.py b/packages/google-cloud-source-context/noxfile.py index 02e8762b9581..500cc452a4d7 100644 --- a/packages/google-cloud-source-context/noxfile.py +++ b/packages/google-cloud-source-context/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-spanner-dbapi-driver/noxfile.py b/packages/google-cloud-spanner-dbapi-driver/noxfile.py index 2fedee7ee5af..7e90464036a9 100644 --- a/packages/google-cloud-spanner-dbapi-driver/noxfile.py +++ b/packages/google-cloud-spanner-dbapi-driver/noxfile.py @@ -482,23 +482,61 @@ def prerelease_deps(session): # the `core_dependencies_from_source` list in the `core_deps_from_source` # nox session should also be updated. prerel_deps = [ - "spannerlib-python", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "grpcio>=1.75.1" if session.python >= "3.12" else "grpcio<=1.62.2", + "grpcio-status", + "protobuf", + "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "spannerlib-python": "google.cloud.spannerlib", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -561,11 +599,25 @@ def core_deps_from_source(session): # added to the list below so that it is installed from source, rather than PyPI # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. - core_dependencies_from_source = [] + core_dependencies_from_source = [ + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", + ] + + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-spanner/noxfile.py b/packages/google-cloud-spanner/noxfile.py index dd6db62994e8..436d549ffe55 100644 --- a/packages/google-cloud-spanner/noxfile.py +++ b/packages/google-cloud-spanner/noxfile.py @@ -606,40 +606,67 @@ def prerelease_deps(session, protobuf_implementation, database_dialect): session.install(*constraints_deps) prerel_deps = [ - "protobuf", - # dependency of grpc - "six", - "grpc-google-iam-v1", "googleapis-common-protos", - "grpcio", - "grpcio-status", "google-api-core", "google-auth", + "grpc-google-iam-v1", + "grpcio>=1.75.1" if session.python >= "3.12" else "grpcio<=1.62.2", + "grpcio-status", + "protobuf", "proto-plus", - "google-cloud-testutils", - # dependencies of google-cloud-testutils" - "click", - # dependency of google-auth - "cffi", - "cryptography", - "cachetools", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--upgrade", dep) - - # Remaining dependencies - other_deps = [ - "requests", - ] - session.install(*other_deps) + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): + print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) - # Print out prerelease package versions - session.run( - "python", "-c", "import google.protobuf; print(google.protobuf.__version__)" - ) - session.run("python", "-c", "import grpc; print(grpc.__version__)") - session.run("python", "-c", "import google.auth; print(google.auth.__version__)") + if version_namespace: + session.run( + "python", + "-c", + f"import {version_namespace}; print({version_namespace}.__version__)", + ) session.run( "py.test", @@ -828,16 +855,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-speech/noxfile.py b/packages/google-cloud-speech/noxfile.py index c4fc8ae4963b..397ca95cb00b 100644 --- a/packages/google-cloud-speech/noxfile.py +++ b/packages/google-cloud-speech/noxfile.py @@ -512,24 +512,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -599,16 +626,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-storage-control/noxfile.py b/packages/google-cloud-storage-control/noxfile.py index 486481fbad7b..1c7b9cfab548 100644 --- a/packages/google-cloud-storage-control/noxfile.py +++ b/packages/google-cloud-storage-control/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-storage-transfer/noxfile.py b/packages/google-cloud-storage-transfer/noxfile.py index 983fc8778809..c85926069955 100644 --- a/packages/google-cloud-storage-transfer/noxfile.py +++ b/packages/google-cloud-storage-transfer/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-storage/noxfile.py b/packages/google-cloud-storage/noxfile.py index 25c24ccb8b6f..6fe377782654 100644 --- a/packages/google-cloud-storage/noxfile.py +++ b/packages/google-cloud-storage/noxfile.py @@ -583,29 +583,60 @@ def prerelease_deps(session, protobuf_implementation): session.install("-e", ".[protobuf, tracing]") prerel_deps = [ + "googleapis-common-protos", "google-api-core", - # Exclude google-auth 3.0.0.dev0 which was yanked - "google-auth!=3.0.0.dev0", - "google-cloud-core", - "google-crc32c", - "google-resumable-media", - "opentelemetry-api", + "google-auth", + "grpc-google-iam-v1", + "grpcio>=1.75.1" if session.python >= "3.12" else "grpcio<=1.62.2", + "grpcio-status", "protobuf", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. package_namespaces = { "google-api-core": "google.api_core", "google-auth": "google.auth", - "google-cloud-core": "google.cloud.version", - "opentelemetry-api": "opentelemetry.version", + "grpcio": "grpc", "protobuf": "google.protobuf", + "proto-plus": "proto", } - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--upgrade", dep) + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") - - version_namespace = package_namespaces.get(dep) + version_namespace = package_namespaces.get(pkg_name) if version_namespace: session.run( @@ -613,9 +644,6 @@ def prerelease_deps(session, protobuf_implementation): "-c", f"import {version_namespace}; print({version_namespace}.__version__)", ) - # Remaining dependencies - other_deps = ["requests", "pyopenssl"] - session.install(*other_deps) session.run( "py.test", @@ -691,16 +719,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-storagebatchoperations/noxfile.py b/packages/google-cloud-storagebatchoperations/noxfile.py index 9ab564421431..8524b56ed318 100644 --- a/packages/google-cloud-storagebatchoperations/noxfile.py +++ b/packages/google-cloud-storagebatchoperations/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-storageinsights/noxfile.py b/packages/google-cloud-storageinsights/noxfile.py index 4e819fe68c16..6ad4f980ecd6 100644 --- a/packages/google-cloud-storageinsights/noxfile.py +++ b/packages/google-cloud-storageinsights/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-support/noxfile.py b/packages/google-cloud-support/noxfile.py index 243b0e5b0f22..44c0ead65cde 100644 --- a/packages/google-cloud-support/noxfile.py +++ b/packages/google-cloud-support/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-talent/noxfile.py b/packages/google-cloud-talent/noxfile.py index a1a6cc9afe4e..c7fe5818c3c7 100644 --- a/packages/google-cloud-talent/noxfile.py +++ b/packages/google-cloud-talent/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-tasks/noxfile.py b/packages/google-cloud-tasks/noxfile.py index 93f682bd1119..be9c38a6967b 100644 --- a/packages/google-cloud-tasks/noxfile.py +++ b/packages/google-cloud-tasks/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-telcoautomation/noxfile.py b/packages/google-cloud-telcoautomation/noxfile.py index 8619575986fc..8568b97d0448 100644 --- a/packages/google-cloud-telcoautomation/noxfile.py +++ b/packages/google-cloud-telcoautomation/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-testutils/noxfile.py b/packages/google-cloud-testutils/noxfile.py index 0e86a298190e..441873ae29cb 100644 --- a/packages/google-cloud-testutils/noxfile.py +++ b/packages/google-cloud-testutils/noxfile.py @@ -303,18 +303,61 @@ def prerelease_deps(session): # the `core_dependencies_from_source` list in the `core_deps_from_source` # nox session should also be updated. prerel_deps = [ + "googleapis-common-protos", + "google-api-core", "google-auth", + "grpc-google-iam-v1", + "grpcio>=1.75.1" if session.python >= "3.12" else "grpcio<=1.62.2", + "grpcio-status", + "protobuf", + "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - package_namespaces = { - "google-auth": "google.auth", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -372,12 +415,24 @@ def core_deps_from_source(session): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - f"{CURRENT_DIRECTORY}/../google-auth", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-texttospeech/noxfile.py b/packages/google-cloud-texttospeech/noxfile.py index b6595f37b11f..d03710954927 100644 --- a/packages/google-cloud-texttospeech/noxfile.py +++ b/packages/google-cloud-texttospeech/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-tpu/noxfile.py b/packages/google-cloud-tpu/noxfile.py index b1e138c76761..2ba7fe3e7de6 100644 --- a/packages/google-cloud-tpu/noxfile.py +++ b/packages/google-cloud-tpu/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-trace/noxfile.py b/packages/google-cloud-trace/noxfile.py index bdc087de6a9a..7ce9d21e6e47 100644 --- a/packages/google-cloud-trace/noxfile.py +++ b/packages/google-cloud-trace/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-translate/noxfile.py b/packages/google-cloud-translate/noxfile.py index bfb500077117..bef5bcd58fdc 100644 --- a/packages/google-cloud-translate/noxfile.py +++ b/packages/google-cloud-translate/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-vectorsearch/noxfile.py b/packages/google-cloud-vectorsearch/noxfile.py index 034c9865c4bc..b7e7756be130 100644 --- a/packages/google-cloud-vectorsearch/noxfile.py +++ b/packages/google-cloud-vectorsearch/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-video-live-stream/noxfile.py b/packages/google-cloud-video-live-stream/noxfile.py index e0e6b278aa58..a02e43ecd00d 100644 --- a/packages/google-cloud-video-live-stream/noxfile.py +++ b/packages/google-cloud-video-live-stream/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-video-stitcher/noxfile.py b/packages/google-cloud-video-stitcher/noxfile.py index 9390e58db0cb..395223981e34 100644 --- a/packages/google-cloud-video-stitcher/noxfile.py +++ b/packages/google-cloud-video-stitcher/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-video-transcoder/noxfile.py b/packages/google-cloud-video-transcoder/noxfile.py index 96163f1cab4c..6f45ce59b5d3 100644 --- a/packages/google-cloud-video-transcoder/noxfile.py +++ b/packages/google-cloud-video-transcoder/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-videointelligence/noxfile.py b/packages/google-cloud-videointelligence/noxfile.py index 2f016518235d..b4a0fb7b99d5 100644 --- a/packages/google-cloud-videointelligence/noxfile.py +++ b/packages/google-cloud-videointelligence/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-vision/noxfile.py b/packages/google-cloud-vision/noxfile.py index e4f47dc0acdb..86d100a34d71 100644 --- a/packages/google-cloud-vision/noxfile.py +++ b/packages/google-cloud-vision/noxfile.py @@ -512,24 +512,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -599,16 +626,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-visionai/noxfile.py b/packages/google-cloud-visionai/noxfile.py index b1c8a8cd5c33..8cc3f6621d21 100644 --- a/packages/google-cloud-visionai/noxfile.py +++ b/packages/google-cloud-visionai/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-vm-migration/noxfile.py b/packages/google-cloud-vm-migration/noxfile.py index 5c4d7cf6a3f5..7a2cb751acaa 100644 --- a/packages/google-cloud-vm-migration/noxfile.py +++ b/packages/google-cloud-vm-migration/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-vmwareengine/noxfile.py b/packages/google-cloud-vmwareengine/noxfile.py index 944adaac0a87..6fd73d9c462e 100644 --- a/packages/google-cloud-vmwareengine/noxfile.py +++ b/packages/google-cloud-vmwareengine/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-vpc-access/noxfile.py b/packages/google-cloud-vpc-access/noxfile.py index 057e6f09cded..243de31c6009 100644 --- a/packages/google-cloud-vpc-access/noxfile.py +++ b/packages/google-cloud-vpc-access/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-webrisk/noxfile.py b/packages/google-cloud-webrisk/noxfile.py index 53677aef7333..ca1eeb142129 100644 --- a/packages/google-cloud-webrisk/noxfile.py +++ b/packages/google-cloud-webrisk/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-websecurityscanner/noxfile.py b/packages/google-cloud-websecurityscanner/noxfile.py index 784e71d26f0c..45653ab9a33b 100644 --- a/packages/google-cloud-websecurityscanner/noxfile.py +++ b/packages/google-cloud-websecurityscanner/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-workflows/noxfile.py b/packages/google-cloud-workflows/noxfile.py index b88d9a4bef6e..ba2d22fcbd1b 100644 --- a/packages/google-cloud-workflows/noxfile.py +++ b/packages/google-cloud-workflows/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-workloadmanager/noxfile.py b/packages/google-cloud-workloadmanager/noxfile.py index af56aceaccef..b8f8d6c610e1 100644 --- a/packages/google-cloud-workloadmanager/noxfile.py +++ b/packages/google-cloud-workloadmanager/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-cloud-workstations/noxfile.py b/packages/google-cloud-workstations/noxfile.py index ce8554e17680..eaeba47f0707 100644 --- a/packages/google-cloud-workstations/noxfile.py +++ b/packages/google-cloud-workstations/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-geo-type/noxfile.py b/packages/google-geo-type/noxfile.py index ff5204010c49..756b8f5f18cc 100644 --- a/packages/google-geo-type/noxfile.py +++ b/packages/google-geo-type/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-maps-addressvalidation/noxfile.py b/packages/google-maps-addressvalidation/noxfile.py index 91cc20b0d634..99fc37b27362 100644 --- a/packages/google-maps-addressvalidation/noxfile.py +++ b/packages/google-maps-addressvalidation/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-maps-areainsights/noxfile.py b/packages/google-maps-areainsights/noxfile.py index b79cead883de..f202c4701fde 100644 --- a/packages/google-maps-areainsights/noxfile.py +++ b/packages/google-maps-areainsights/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-maps-fleetengine-delivery/noxfile.py b/packages/google-maps-fleetengine-delivery/noxfile.py index 7785225729bd..9f6b7c0e3862 100644 --- a/packages/google-maps-fleetengine-delivery/noxfile.py +++ b/packages/google-maps-fleetengine-delivery/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-maps-fleetengine/noxfile.py b/packages/google-maps-fleetengine/noxfile.py index 3762a7d5c2cf..8f6063d8e9e7 100644 --- a/packages/google-maps-fleetengine/noxfile.py +++ b/packages/google-maps-fleetengine/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-maps-geocode/noxfile.py b/packages/google-maps-geocode/noxfile.py index 87cd361917df..5ac9fbc7de1b 100644 --- a/packages/google-maps-geocode/noxfile.py +++ b/packages/google-maps-geocode/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-maps-mapsplatformdatasets/noxfile.py b/packages/google-maps-mapsplatformdatasets/noxfile.py index 50e20a57dc1a..f3d475a2b688 100644 --- a/packages/google-maps-mapsplatformdatasets/noxfile.py +++ b/packages/google-maps-mapsplatformdatasets/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-maps-navconnect/noxfile.py b/packages/google-maps-navconnect/noxfile.py index c1d1d1eabbf6..5445e22de411 100644 --- a/packages/google-maps-navconnect/noxfile.py +++ b/packages/google-maps-navconnect/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-maps-places/noxfile.py b/packages/google-maps-places/noxfile.py index b9c3f749479d..6bc342b1019b 100644 --- a/packages/google-maps-places/noxfile.py +++ b/packages/google-maps-places/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-maps-routeoptimization/noxfile.py b/packages/google-maps-routeoptimization/noxfile.py index 49b7020937ab..18ce7327cf60 100644 --- a/packages/google-maps-routeoptimization/noxfile.py +++ b/packages/google-maps-routeoptimization/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-maps-routing/noxfile.py b/packages/google-maps-routing/noxfile.py index 4b29a7c44218..c37e2f62ff67 100644 --- a/packages/google-maps-routing/noxfile.py +++ b/packages/google-maps-routing/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-maps-solar/noxfile.py b/packages/google-maps-solar/noxfile.py index 9ec496032792..0611c2f8fc49 100644 --- a/packages/google-maps-solar/noxfile.py +++ b/packages/google-maps-solar/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-shopping-css/noxfile.py b/packages/google-shopping-css/noxfile.py index fadb5e4884db..241cf9809873 100644 --- a/packages/google-shopping-css/noxfile.py +++ b/packages/google-shopping-css/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-shopping-merchant-accounts/noxfile.py b/packages/google-shopping-merchant-accounts/noxfile.py index f4803152aaad..fbc9dd34882a 100644 --- a/packages/google-shopping-merchant-accounts/noxfile.py +++ b/packages/google-shopping-merchant-accounts/noxfile.py @@ -510,24 +510,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -597,16 +624,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-shopping-merchant-conversions/noxfile.py b/packages/google-shopping-merchant-conversions/noxfile.py index a2ccdc965500..19d8592e3f3f 100644 --- a/packages/google-shopping-merchant-conversions/noxfile.py +++ b/packages/google-shopping-merchant-conversions/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-shopping-merchant-datasources/noxfile.py b/packages/google-shopping-merchant-datasources/noxfile.py index 89ea9df0bc2c..959ac540aca7 100644 --- a/packages/google-shopping-merchant-datasources/noxfile.py +++ b/packages/google-shopping-merchant-datasources/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-shopping-merchant-inventories/noxfile.py b/packages/google-shopping-merchant-inventories/noxfile.py index c7068f52812d..9379f34c1eab 100644 --- a/packages/google-shopping-merchant-inventories/noxfile.py +++ b/packages/google-shopping-merchant-inventories/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-shopping-merchant-issueresolution/noxfile.py b/packages/google-shopping-merchant-issueresolution/noxfile.py index 9c47449fc049..cd51958d7297 100644 --- a/packages/google-shopping-merchant-issueresolution/noxfile.py +++ b/packages/google-shopping-merchant-issueresolution/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-shopping-merchant-lfp/noxfile.py b/packages/google-shopping-merchant-lfp/noxfile.py index e99b7f916629..0d52f86639d4 100644 --- a/packages/google-shopping-merchant-lfp/noxfile.py +++ b/packages/google-shopping-merchant-lfp/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-shopping-merchant-notifications/noxfile.py b/packages/google-shopping-merchant-notifications/noxfile.py index 50993ef9d602..a8464a983701 100644 --- a/packages/google-shopping-merchant-notifications/noxfile.py +++ b/packages/google-shopping-merchant-notifications/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-shopping-merchant-ordertracking/noxfile.py b/packages/google-shopping-merchant-ordertracking/noxfile.py index 0124f2e18819..3def6b1650dc 100644 --- a/packages/google-shopping-merchant-ordertracking/noxfile.py +++ b/packages/google-shopping-merchant-ordertracking/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-shopping-merchant-products/noxfile.py b/packages/google-shopping-merchant-products/noxfile.py index a66448fac790..4e5dc1869aeb 100644 --- a/packages/google-shopping-merchant-products/noxfile.py +++ b/packages/google-shopping-merchant-products/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-shopping-merchant-productstudio/noxfile.py b/packages/google-shopping-merchant-productstudio/noxfile.py index de9c809f7fb5..e10caf2d10e0 100644 --- a/packages/google-shopping-merchant-productstudio/noxfile.py +++ b/packages/google-shopping-merchant-productstudio/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-shopping-merchant-promotions/noxfile.py b/packages/google-shopping-merchant-promotions/noxfile.py index 83e0666cf2c0..fa5c521a3bd7 100644 --- a/packages/google-shopping-merchant-promotions/noxfile.py +++ b/packages/google-shopping-merchant-promotions/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-shopping-merchant-quota/noxfile.py b/packages/google-shopping-merchant-quota/noxfile.py index 40d10df20445..40b8f4f0c943 100644 --- a/packages/google-shopping-merchant-quota/noxfile.py +++ b/packages/google-shopping-merchant-quota/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-shopping-merchant-reports/noxfile.py b/packages/google-shopping-merchant-reports/noxfile.py index e784edeef999..f0f9acd13ef8 100644 --- a/packages/google-shopping-merchant-reports/noxfile.py +++ b/packages/google-shopping-merchant-reports/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-shopping-merchant-reviews/noxfile.py b/packages/google-shopping-merchant-reviews/noxfile.py index aadc41691038..e4fc3b9b562f 100644 --- a/packages/google-shopping-merchant-reviews/noxfile.py +++ b/packages/google-shopping-merchant-reviews/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/google-shopping-type/noxfile.py b/packages/google-shopping-type/noxfile.py index 8b22dbc65b39..d23723313da2 100644 --- a/packages/google-shopping-type/noxfile.py +++ b/packages/google-shopping-type/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/googleapis-common-protos/noxfile.py b/packages/googleapis-common-protos/noxfile.py index ce20cdcfcad8..b9c6e914c5c5 100644 --- a/packages/googleapis-common-protos/noxfile.py +++ b/packages/googleapis-common-protos/noxfile.py @@ -486,30 +486,57 @@ def prerelease_deps(session, protobuf_implementation): "google-api-core", "google-auth", "grpc-google-iam-v1", - "grpcio", + "grpcio>=1.75.1" if session.python >= "3.12" else "grpcio<=1.62.2", "grpcio-status", "protobuf", "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -581,16 +608,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - f"{CURRENT_DIRECTORY}/../googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - f"{CURRENT_DIRECTORY}/../grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/grafeas/noxfile.py b/packages/grafeas/noxfile.py index fc27aca011f4..0cd63769738a 100644 --- a/packages/grafeas/noxfile.py +++ b/packages/grafeas/noxfile.py @@ -511,24 +511,51 @@ def prerelease_deps(session, protobuf_implementation): "proto-plus", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + if version_namespace: session.run( "python", @@ -598,16 +625,24 @@ def core_deps_from_source(session, protobuf_implementation): # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/grpc-google-iam-v1/noxfile.py b/packages/grpc-google-iam-v1/noxfile.py index 6d0703ed100b..778719e52055 100644 --- a/packages/grpc-google-iam-v1/noxfile.py +++ b/packages/grpc-google-iam-v1/noxfile.py @@ -479,36 +479,67 @@ def prerelease_deps(session, protobuf_implementation): session.install(*constraints_deps) prerel_deps = [ - "protobuf", - # dependency of grpc - "six", - "grpc-google-iam-v1", "googleapis-common-protos", - "grpcio", - "grpcio-status", "google-api-core", "google-auth", + "grpc-google-iam-v1", + "grpcio>=1.75.1" if session.python >= "3.12" else "grpcio<=1.62.2", + "grpcio-status", + "protobuf", "proto-plus", - "google-cloud-testutils", - # dependencies of google-cloud-testutils" - "click", - ] - - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--upgrade", dep) - - # Remaining dependencies - other_deps = [ - "requests", ] - session.install(*other_deps) - # Print out prerelease package versions - session.run( - "python", "-c", "import google.protobuf; print(google.protobuf.__version__)" - ) - session.run("python", "-c", "import grpc; print(grpc.__version__)") - session.run("python", "-c", "import google.auth; print(google.auth.__version__)") + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): + print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) + + if version_namespace: + session.run( + "python", + "-c", + f"import {version_namespace}; print({version_namespace}.__version__)", + ) session.run( "py.test", @@ -567,15 +598,24 @@ def core_deps_from_source(session, protobuf_implementation): session.install(*constraints_deps) core_dependencies_from_source = [ - f"{CURRENT_DIRECTORY}/../google-api-core", - f"{CURRENT_DIRECTORY}/../google-auth", - f"{CURRENT_DIRECTORY}/../googleapis-common-protos", - f"{CURRENT_DIRECTORY}/../grpc-google-iam-v1", - f"{CURRENT_DIRECTORY}/../proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--ignore-installed", "--no-deps") + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", diff --git a/packages/pandas-gbq/noxfile.py b/packages/pandas-gbq/noxfile.py index b850a2a74e0a..2a18f4b08c8d 100644 --- a/packages/pandas-gbq/noxfile.py +++ b/packages/pandas-gbq/noxfile.py @@ -542,18 +542,24 @@ def core_deps_from_source(session, protobuf_implementation): install_unittest_dependencies(session, "-c", constraints_path) core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] - for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent - tests_path = os.path.join("tests", "unit") + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + + for dep in core_dependencies_from_source: + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", "--quiet", diff --git a/packages/proto-plus/noxfile.py b/packages/proto-plus/noxfile.py index 16561f17154c..86da86c410bc 100644 --- a/packages/proto-plus/noxfile.py +++ b/packages/proto-plus/noxfile.py @@ -113,10 +113,34 @@ def prerelease_deps(session, implementation): "googleapis-common-protos", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--upgrade", dep) + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--upgrade") session.install("--pre", "--upgrade", "protobuf") + # Print out prerelease package versions session.run( "python", "-c", "import google.protobuf; print(google.protobuf.__version__)" @@ -164,19 +188,27 @@ def core_deps_from_source(session, implementation): # Install the package without dependencies session.install("-e", ".", "--no-deps") + # Dynamically locate the monorepo packages/ directory + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + # TODO(https://github.com/googleapis/gapic-generator-python/issues/2357): `protobuf` should be # added to the list below so that it is installed from source, rather than PyPI # Note: If a dependency is added to the `core_dependencies_from_source` list, # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. core_dependencies_from_source = [ - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", + "google-api-core", # dependency of google-api-core - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", + "googleapis-common-protos", ] + # Batch install the local directories to eliminate pip network overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") + for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + print(f"Installed {dep} locally from {deps_dir / dep}") # TODO(https://github.com/googleapis/google-cloud-python/issues/15115): Install protobuf from source at HEAD session.install("--pre", "--upgrade", "protobuf") diff --git a/packages/sqlalchemy-bigquery/noxfile.py b/packages/sqlalchemy-bigquery/noxfile.py index 2e171cc31444..5ba7f2145d63 100644 --- a/packages/sqlalchemy-bigquery/noxfile.py +++ b/packages/sqlalchemy-bigquery/noxfile.py @@ -607,39 +607,67 @@ def prerelease_deps(session, protobuf_implementation): session.install(*constraints_deps) prerel_deps = [ - "protobuf", - "sqlalchemy", - # dependency of grpc - "six", - "grpc-google-iam-v1", "googleapis-common-protos", - "grpcio", - "grpcio-status", "google-api-core", "google-auth", + "grpc-google-iam-v1", + "grpcio>=1.75.1" if session.python >= "3.12" else "grpcio<=1.62.2", + "grpcio-status", + "protobuf", "proto-plus", - "google-cloud-testutils", - # dependencies of google-cloud-testutils" - "click", ] - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--upgrade", dep) - - # Remaining dependencies - other_deps = [ - "requests", - ] - session.install(*other_deps) - - session.run("python", "-m", "pip", "freeze") + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Extract the base package name, safely ignoring version bounds and spaces + # (e.g., "grpcio>=1.75.1" becomes "grpcio") + parsed_deps = { + dep: re.match(r"^([a-zA-Z0-9_-]+)", dep).group(1) + for dep in prerel_deps + } + + # Dynamically sort local packages vs PyPI dependencies + local_paths = [] + pypi_deps = [] + + for dep, pkg_name in parsed_deps.items(): + if (deps_dir / pkg_name).exists(): + local_paths.append(str(deps_dir / pkg_name)) + else: + pypi_deps.append(dep) + + # Batch pip installations for maximum speed + if local_paths: + session.install(*local_paths, "--no-deps", "--ignore-installed") + if pypi_deps: + session.install(*pypi_deps, "--pre", "--no-deps", "--ignore-installed") + + # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status` + # to the dictionary below once this bug is fixed. + # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add + # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below + # once this bug is fixed. + package_namespaces = { + "google-api-core": "google.api_core", + "google-auth": "google.auth", + "grpcio": "grpc", + "protobuf": "google.protobuf", + "proto-plus": "proto", + } + + # Reuse the parsed names for logging and version verification + for dep, pkg_name in parsed_deps.items(): + print(f"Installed {dep}") + version_namespace = package_namespaces.get(pkg_name) - # Print out prerelease package versions - session.run( - "python", "-c", "import google.protobuf; print(google.protobuf.__version__)" - ) - session.run("python", "-c", "import grpc; print(grpc.__version__)") - session.run("python", "-c", "import google.auth; print(google.auth.__version__)") + if version_namespace: + session.run( + "python", + "-c", + f"import {version_namespace}; print({version_namespace}.__version__)", + ) session.run( "py.test", @@ -714,18 +742,24 @@ def core_deps_from_source(session, protobuf_implementation): session.install("-e", install_target, "-c", constraints_path) core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] - for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") - tests_path = os.path.join("tests", "unit") + for dep in core_dependencies_from_source: + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", "--quiet", diff --git a/packages/sqlalchemy-spanner/noxfile.py b/packages/sqlalchemy-spanner/noxfile.py index 1dbbb74bea29..3115569ba978 100644 --- a/packages/sqlalchemy-spanner/noxfile.py +++ b/packages/sqlalchemy-spanner/noxfile.py @@ -425,18 +425,24 @@ def core_deps_from_source(session, protobuf_implementation): session.install("opentelemetry-instrumentation") core_dependencies_from_source = [ - "googleapis-common-protos @ git+https://github.com/googleapis/google-cloud-python#egg=googleapis-common-protos&subdirectory=packages/googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/google-cloud-python#egg=google-api-core&subdirectory=packages/google-api-core", - "google-auth @ git+https://github.com/googleapis/google-cloud-python#egg=google-auth&subdirectory=packages/google-auth", - "grpc-google-iam-v1 @ git+https://github.com/googleapis/google-cloud-python#egg=grpc-google-iam-v1&subdirectory=packages/grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/google-cloud-python#egg=proto-plus&subdirectory=packages/proto-plus", + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpc-google-iam-v1", + "proto-plus", ] - for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") + deps_dir = CURRENT_DIRECTORY.parent + while deps_dir.name != "packages" and deps_dir.parent != deps_dir: + deps_dir = deps_dir.parent + + # Batch the pip installation to avoid sequential overhead + dep_paths = [str(deps_dir / dep) for dep in core_dependencies_from_source] + session.install(*dep_paths, "--no-deps", "--ignore-installed") - tests_path = os.path.join("tests", "unit") + for dep in core_dependencies_from_source: + dep_path = str(deps_dir / dep) + print(f"Installed {dep} locally from {dep_path}") session.run( "py.test", "--quiet",