Skip to content

fix: use requirements files with hashes instead of constraints #17512

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
parthea wants to merge 8 commits into
base: main
Choose a base branch
from
Draft

fix: use requirements files with hashes instead of constraints #17512

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
2f46631
chore: rename constraints files to requirements templates
parthea Jun 18, 2026
39dcbaa
fix: use requirements files with hashes instead of constraints
parthea Jun 18, 2026
a8af701
work in progress
parthea Jun 18, 2026
06b45d8
work in progress
parthea Jun 18, 2026
9ef477e
work in progress
parthea Jun 18, 2026
db5edde
work in progress
parthea Jun 18, 2026
b9a7114
work in progress
parthea Jun 18, 2026
855fc26
work in progress
parthea Jun 18, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
170 changes: 58 additions & 112 deletions packages/gapic-generator/gapic/templates/noxfile.py.j2
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,38 +41,12 @@ PREVIEW_PYTHON_VERSION = "3.14"

CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute()

if (CURRENT_DIRECTORY / "testing").exists():
LOWER_BOUND_CONSTRAINTS_FILE = (
CURRENT_DIRECTORY / "testing" / f"constraints-{ALL_PYTHON[0]}.txt"
)
else:
LOWER_BOUND_CONSTRAINTS_FILE = CURRENT_DIRECTORY / "constraints.txt"
LOWER_BOUND_REQUIREMENTS_FILE = (
CURRENT_DIRECTORY / "testing" / f"requirements-{ALL_PYTHON[0]}.txt"
)
PACKAGE_NAME = "{{ api.naming.warehouse_package_name }}"

UNIT_TEST_STANDARD_DEPENDENCIES = [
"mock",
"asyncmock",
"pytest",
"pytest-cov",
"pytest-asyncio",
]
UNIT_TEST_EXTERNAL_DEPENDENCIES: List[str] = []
UNIT_TEST_LOCAL_DEPENDENCIES: List[str] = []
UNIT_TEST_DEPENDENCIES: List[str] = []
UNIT_TEST_EXTRAS: List[str] = []
UNIT_TEST_EXTRAS_BY_PYTHON: Dict[str, List[str]] = {}

SYSTEM_TEST_PYTHON_VERSIONS: List[str] = ALL_PYTHON
SYSTEM_TEST_STANDARD_DEPENDENCIES = [
"mock",
"pytest",
"google-cloud-testutils",
]
SYSTEM_TEST_EXTERNAL_DEPENDENCIES: List[str] = []
SYSTEM_TEST_LOCAL_DEPENDENCIES: List[str] = []
SYSTEM_TEST_DEPENDENCIES: List[str] = []
SYSTEM_TEST_EXTRAS: List[str] = []
SYSTEM_TEST_EXTRAS_BY_PYTHON: Dict[str, List[str]] = {}

nox.options.sessions = [
"unit",
Expand Down Expand Up @@ -114,33 +88,48 @@ def mypy(session):

@nox.session
def update_lower_bounds(session):
"""Update lower bounds in constraints.txt to match setup.py"""
"""Update lower bounds in requirements.txt to match setup.py"""
session.install("google-cloud-testutils")
session.install(".")

requirements_in = CURRENT_DIRECTORY / "testing" / f"requirements-{ALL_PYTHON[0]}.in"

session.run(
"lower-bound-checker",
"update",
"--package-name",
PACKAGE_NAME,
"--constraints-file",
str(LOWER_BOUND_CONSTRAINTS_FILE),
str(requirements_in),
)

session.install("pip-tools")
session.run(
"pip-compile",
"--allow-unsafe",
"--generate-hashes",
"--no-strip-extras",
"--output-file",
str(LOWER_BOUND_REQUIREMENTS_FILE),
str(requirements_in),
)


@nox.session
def check_lower_bounds(session):
"""Check lower bounds in setup.py are reflected in constraints file"""
"""Check lower bounds in setup.py are reflected in requirements file"""
session.install("google-cloud-testutils")
session.install(".")

requirements_in = CURRENT_DIRECTORY / "testing" / f"requirements-{ALL_PYTHON[0]}.in"

session.run(
"lower-bound-checker",
"check",
"--package-name",
PACKAGE_NAME,
"--constraints-file",
str(LOWER_BOUND_CONSTRAINTS_FILE),
str(requirements_in),
)


Expand Down Expand Up @@ -221,32 +210,12 @@ def lint_setup_py(session):
session.run("python", "setup.py", "check", "--restructuredtext", "--strict")


def install_unittest_dependencies(session, *constraints):
standard_deps = UNIT_TEST_STANDARD_DEPENDENCIES + UNIT_TEST_DEPENDENCIES
session.install(*standard_deps, *constraints)

if UNIT_TEST_EXTERNAL_DEPENDENCIES:
warnings.warn(
"'unit_test_external_dependencies' is deprecated. Instead, please "
"use 'unit_test_dependencies' or 'unit_test_local_dependencies'.",
DeprecationWarning,
)
session.install(*UNIT_TEST_EXTERNAL_DEPENDENCIES, *constraints)

if UNIT_TEST_LOCAL_DEPENDENCIES:
session.install(*UNIT_TEST_LOCAL_DEPENDENCIES, *constraints)

if UNIT_TEST_EXTRAS_BY_PYTHON:
extras = UNIT_TEST_EXTRAS_BY_PYTHON.get(session.python, [])
elif UNIT_TEST_EXTRAS:
extras = UNIT_TEST_EXTRAS
else:
extras = []

if extras:
session.install("-e", f".[{','.join(extras)}]", *constraints)
else:
session.install("-e", ".", *constraints)
def install_unittest_dependencies(session, *requirements):
requirements_path = str(
CURRENT_DIRECTORY / "testing" / f"requirements-{session.python}.txt"
)
session.install("-r", requirements_path)
session.install("-e", ".", "--no-deps")


@nox.session(python=ALL_PYTHON)
Expand All @@ -257,10 +226,10 @@ def install_unittest_dependencies(session, *constraints):
def unit(session, protobuf_implementation):
# Install all test dependencies, then install this package in-place.

constraints_path = str(
CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt"
requirements_path = str(
CURRENT_DIRECTORY / "testing" / f"requirements-{session.python}.txt"
)
install_unittest_dependencies(session, "-c", constraints_path)
install_unittest_dependencies(session, "-r", requirements_path)

# Run py.test against the unit tests.
session.run(
Expand All @@ -286,42 +255,11 @@ def unit(session, protobuf_implementation):
{# TODO(https://github.com/googleapis/gapic-generator-python/issues/2201) Add a `unit_rest_async` nox session to run tests with [async_rest] extra installed. #}


def install_systemtest_dependencies(session, *constraints):
{# Note that grpcio<=1.62.2 works on Python 3.11 but fails on 3.12+ because it requires pkg_resources for its source build. #}
if session.python >= "3.12":
session.install("--pre", "grpcio>=1.75.1")
else:
session.install("--pre", "grpcio<=1.62.2")

session.install(*SYSTEM_TEST_STANDARD_DEPENDENCIES, *constraints)

if SYSTEM_TEST_EXTERNAL_DEPENDENCIES:
session.install(*SYSTEM_TEST_EXTERNAL_DEPENDENCIES, *constraints)

if SYSTEM_TEST_LOCAL_DEPENDENCIES:
session.install("-e", *SYSTEM_TEST_LOCAL_DEPENDENCIES, *constraints)

if SYSTEM_TEST_DEPENDENCIES:
session.install("-e", *SYSTEM_TEST_DEPENDENCIES, *constraints)

if SYSTEM_TEST_EXTRAS_BY_PYTHON:
extras = SYSTEM_TEST_EXTRAS_BY_PYTHON.get(session.python, [])
elif SYSTEM_TEST_EXTRAS:
extras = SYSTEM_TEST_EXTRAS
else:
extras = []

if extras:
session.install("-e", f".[{','.join(extras)}]", *constraints)
else:
session.install("-e", ".", *constraints)


@nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS)
def system(session):
"""Run the system test suite."""
constraints_path = str(
CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt"
requirements_path = str(
CURRENT_DIRECTORY / "testing" / f"requirements-{session.python}.txt"
)
system_test_path = os.path.join("tests", "system.py")
system_test_folder_path = os.path.join("tests", "system")
Expand All @@ -339,7 +277,7 @@ def system(session):
if not system_test_exists and not system_test_folder_exists:
session.skip("System tests were not found")

install_systemtest_dependencies(session, "-c", constraints_path)
install_systemtest_dependencies(session, "-r", requirements_path)

# Run py.test against the system tests.
if system_test_exists:
Expand Down Expand Up @@ -474,23 +412,23 @@ def prerelease_deps(session, protobuf_implementation):

# Because we test minimum dependency versions on the minimum Python
# version, the first version we test with in the unit tests sessions has a
# constraints file containing all dependencies and extras.
# requirements file containing all dependencies and extras.
with open(
CURRENT_DIRECTORY / "testing" / f"constraints-{ALL_PYTHON[0]}.txt",
CURRENT_DIRECTORY / "testing" / f"requirements-{ALL_PYTHON[0]}.txt",
encoding="utf-8",
) as constraints_file:
constraints_text = constraints_file.read()
) as requirements_file:
requirements_text = requirements_file.read()

# Ignore leading whitespace and comment lines.
constraints_deps = [
requirements_deps = [
match.group(1)
for match in re.finditer(
r"^\s*(\S+)(?===\S+)", constraints_text, flags=re.MULTILINE
r"^\s*(\S+)(?===\S+)", requirements_text, flags=re.MULTILINE
)
]

# Install dependencies specified in `testing/constraints-X.txt`.
session.install(*constraints_deps)
# Install dependencies specified in `testing/requirements-X.txt`.
session.install(*requirements_deps)

# Note: If a dependency is added to the `prerel_deps` list,
# the `core_dependencies_from_source` list in the `core_deps_from_source`
Expand Down Expand Up @@ -567,6 +505,14 @@ def prerelease_deps(session, protobuf_implementation):
)


@nox.session(python=ALL_PYTHON[0])
def security_check_minimum_version(session):
# Do not pin `pip-audit` since we need the latest version to be aware of new vulnerabilities
session.install("pip-audit")
session.run("pip-audit", "--version")
session.run("pip-audit", "-r", CURRENT_DIRECTORY / "testing" / f"requirements-{ALL_PYTHON[0]}.txt")


@nox.session(python=DEFAULT_PYTHON_VERSION)
@nox.parametrize(
"protobuf_implementation",
Expand All @@ -586,23 +532,23 @@ def core_deps_from_source(session, protobuf_implementation):

# Because we test minimum dependency versions on the minimum Python
# version, the first version we test with in the unit tests sessions has a
# constraints file containing all dependencies and extras.
# requirements file containing all dependencies and extras.
with open(
CURRENT_DIRECTORY / "testing" / f"constraints-{ALL_PYTHON[0]}.txt",
CURRENT_DIRECTORY / "testing" / f"requirements-{ALL_PYTHON[0]}.txt",
encoding="utf-8",
) as constraints_file:
constraints_text = constraints_file.read()
) as requirements_file:
requirements_text = requirements_file.read()

# Ignore leading whitespace and comment lines.
constraints_deps = [
requirements_deps = [
match.group(1)
for match in re.finditer(
r"^\s*(\S+)(?===\S+)", constraints_text, flags=re.MULTILINE
r"^\s*(\S+)(?===\S+)", requirements_text, flags=re.MULTILINE
)
]

# Install dependencies specified in `testing/constraints-X.txt`.
session.install(*constraints_deps)
# Install dependencies specified in `testing/requirements-X.txt`.
session.install(*requirements_deps)

# TODO(https://github.com/googleapis/gapic-generator-python/issues/2358): `grpcio` and
# `grpcio-status` should be added to the list below so that they are installed from source,
Expand Down
2 changes: 1 addition & 1 deletion packages/gapic-generator/gapic/templates/setup.py.j2
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ dependencies = [
{# Explicitly exclude protobuf versions mentioned in https://cloud.google.com/support/bulletins#GCP-2022-019 #}
"protobuf >= 6.33.5, < 8.0.0",
{% for package_tuple, package_info in pypi_packages.items() %}
{# Quick check to make sure `package_info.package_name` is not the package being generated so we don't circularly include this package in its own constraints file. #}
{# Quick check to make sure `package_info.package_name` is not the package being generated so we don't circularly include this package in its own requirements file. #}
{% if api.naming.warehouse_package_name != package_info.package_name %}
{% if api.requires_package(package_tuple) %}
"{{ package_info.package_name }} >= {{ package_info.lower_bound }}, <{{ package_info.upper_bound }}",
Expand Down
11 changes: 9 additions & 2 deletions ...templates/testing/_default_constraints.j2 → ...emplates/testing/_default_requirements.j2
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
{% set rest_async_io_enabled = api.all_library_settings[api.naming.proto_package].python_settings.experimental_features.rest_async_io_enabled %}
{% from '_pypi_packages.j2' import pypi_packages %}
# This constraints file is required for unit tests.
# This requirements file is required for unit tests.
# List all library dependencies and extras in this file.
google-api-core
google-auth
Expand All @@ -8,8 +9,14 @@ proto-plus
protobuf
# cryptography is a direct dependency of google-auth
cryptography
# pytest-asyncio is needed for tests otherwise async tests are skipped
pytest-asyncio
{% if rest_async_io_enabled %}
aiohttp
requests
{% endif %}
{% for package_tuple, package_info in pypi_packages.items() %}
{# Quick check to make sure `package_info.package_name` is not the package being generated so we don't circularly include this package in its own constraints file. #}
{# Quick check to make sure `package_info.package_name` is not the package being generated so we don't circularly include this package in its own requirements file. #}
{% if api.naming.warehouse_package_name != package_info.package_name %}
{% if api.requires_package(package_tuple) %}
{{ package_info.package_name }}
Expand Down
Loading
Loading