Skip to content

Commit b886feb

Browse files
partheavchudnov-ggcf-owl-bot[bot]
authored
fix: add support for protobuf 5.x (googleapis#644)
* fix: add support for protobuf 5.x * remove pin for types-protobuf * remove pytest from noxfile * refactor common code * Refactor Co-authored-by: Victor Chudnovsky <vchudnov@google.com> * run black * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * mypy * run pre-release test against all python versions * filter warning --------- Co-authored-by: Victor Chudnovsky <vchudnov@google.com> Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>
1 parent 6819fb4 commit b886feb

File tree

9 files changed

+141
-52
lines changed

9 files changed

+141
-52
lines changed

‎packages/google-api-core/.github/workflows/unittest.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ jobs:
1111
runs-on: ubuntu-latest
1212
strategy:
1313
matrix:
14-
option: ["", "_grpc_gcp", "_wo_grpc"]
14+
option: ["", "_grpc_gcp", "_wo_grpc", "_with_prerelease_deps"]
1515
python:
1616
- "3.7"
1717
- "3.8"

‎packages/google-api-core/google/api_core/operations_v1/__init__.py‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,9 @@
1414

1515
"""Package for interacting with the google.longrunning.operations meta-API."""
1616

17-
from google.api_core.operations_v1.abstract_operations_client import AbstractOperationsClient
17+
from google.api_core.operations_v1.abstract_operations_client import (
18+
AbstractOperationsClient,
19+
)
1820
from google.api_core.operations_v1.operations_async_client import OperationsAsyncClient
1921
from google.api_core.operations_v1.operations_client import OperationsClient
2022
from google.api_core.operations_v1.transports.rest import OperationsRestTransport
@@ -23,5 +25,5 @@
2325
"AbstractOperationsClient",
2426
"OperationsAsyncClient",
2527
"OperationsClient",
26-
"OperationsRestTransport"
28+
"OperationsRestTransport",
2729
]

‎packages/google-api-core/google/api_core/operations_v1/transports/base.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ def __init__(
4545
self,
4646
*,
4747
host: str = DEFAULT_HOST,
48-
credentials: ga_credentials.Credentials = None,
48+
credentials: Optional[ga_credentials.Credentials] = None,
4949
credentials_file: Optional[str] = None,
5050
scopes: Optional[Sequence[str]] = None,
5151
quota_project_id: Optional[str] = None,

‎packages/google-api-core/google/api_core/operations_v1/transports/rest.py‎

Lines changed: 41 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -29,9 +29,13 @@
2929
from google.longrunning import operations_pb2 # type: ignore
3030
from google.protobuf import empty_pb2 # type: ignore
3131
from google.protobuf import json_format # type: ignore
32+
import google.protobuf
33+
3234
import grpc
3335
from .base import DEFAULT_CLIENT_INFO as BASE_DEFAULT_CLIENT_INFO, OperationsTransport
3436

37+
PROTOBUF_VERSION = google.protobuf.__version__
38+
3539
OptionalRetry = Union[retries.Retry, object]
3640

3741
DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo(
@@ -66,7 +70,7 @@ def __init__(
6670
self,
6771
*,
6872
host: str = "longrunning.googleapis.com",
69-
credentials: ga_credentials.Credentials = None,
73+
credentials: Optional[ga_credentials.Credentials] = None,
7074
credentials_file: Optional[str] = None,
7175
scopes: Optional[Sequence[str]] = None,
7276
client_cert_source_for_mtls: Optional[Callable[[], Tuple[bytes, bytes]]] = None,
@@ -184,11 +188,7 @@ def _list_operations(
184188
"google.longrunning.Operations.ListOperations"
185189
]
186190

187-
request_kwargs = json_format.MessageToDict(
188-
request,
189-
preserving_proto_field_name=True,
190-
including_default_value_fields=True,
191-
)
191+
request_kwargs = self._convert_protobuf_message_to_dict(request)
192192
transcoded_request = path_template.transcode(http_options, **request_kwargs)
193193

194194
uri = transcoded_request["uri"]
@@ -199,7 +199,6 @@ def _list_operations(
199199
json_format.ParseDict(transcoded_request["query_params"], query_params_request)
200200
query_params = json_format.MessageToDict(
201201
query_params_request,
202-
including_default_value_fields=False,
203202
preserving_proto_field_name=False,
204203
use_integers_for_enums=False,
205204
)
@@ -265,11 +264,7 @@ def _get_operation(
265264
"google.longrunning.Operations.GetOperation"
266265
]
267266

268-
request_kwargs = json_format.MessageToDict(
269-
request,
270-
preserving_proto_field_name=True,
271-
including_default_value_fields=True,
272-
)
267+
request_kwargs = self._convert_protobuf_message_to_dict(request)
273268
transcoded_request = path_template.transcode(http_options, **request_kwargs)
274269

275270
uri = transcoded_request["uri"]
@@ -280,7 +275,6 @@ def _get_operation(
280275
json_format.ParseDict(transcoded_request["query_params"], query_params_request)
281276
query_params = json_format.MessageToDict(
282277
query_params_request,
283-
including_default_value_fields=False,
284278
preserving_proto_field_name=False,
285279
use_integers_for_enums=False,
286280
)
@@ -339,11 +333,7 @@ def _delete_operation(
339333
"google.longrunning.Operations.DeleteOperation"
340334
]
341335

342-
request_kwargs = json_format.MessageToDict(
343-
request,
344-
preserving_proto_field_name=True,
345-
including_default_value_fields=True,
346-
)
336+
request_kwargs = self._convert_protobuf_message_to_dict(request)
347337
transcoded_request = path_template.transcode(http_options, **request_kwargs)
348338

349339
uri = transcoded_request["uri"]
@@ -354,7 +344,6 @@ def _delete_operation(
354344
json_format.ParseDict(transcoded_request["query_params"], query_params_request)
355345
query_params = json_format.MessageToDict(
356346
query_params_request,
357-
including_default_value_fields=False,
358347
preserving_proto_field_name=False,
359348
use_integers_for_enums=False,
360349
)
@@ -411,19 +400,14 @@ def _cancel_operation(
411400
"google.longrunning.Operations.CancelOperation"
412401
]
413402

414-
request_kwargs = json_format.MessageToDict(
415-
request,
416-
preserving_proto_field_name=True,
417-
including_default_value_fields=True,
418-
)
403+
request_kwargs = self._convert_protobuf_message_to_dict(request)
419404
transcoded_request = path_template.transcode(http_options, **request_kwargs)
420405

421406
# Jsonify the request body
422407
body_request = operations_pb2.CancelOperationRequest()
423408
json_format.ParseDict(transcoded_request["body"], body_request)
424409
body = json_format.MessageToDict(
425410
body_request,
426-
including_default_value_fields=False,
427411
preserving_proto_field_name=False,
428412
use_integers_for_enums=False,
429413
)
@@ -435,7 +419,6 @@ def _cancel_operation(
435419
json_format.ParseDict(transcoded_request["query_params"], query_params_request)
436420
query_params = json_format.MessageToDict(
437421
query_params_request,
438-
including_default_value_fields=False,
439422
preserving_proto_field_name=False,
440423
use_integers_for_enums=False,
441424
)
@@ -458,6 +441,38 @@ def _cancel_operation(
458441

459442
return empty_pb2.Empty()
460443

444+
def _convert_protobuf_message_to_dict(
445+
self, message: google.protobuf.message.Message
446+
):
447+
r"""Converts protobuf message to a dictionary.
448+
449+
When the dictionary is encoded to JSON, it conforms to proto3 JSON spec.
450+
451+
Args:
452+
message(google.protobuf.message.Message): The protocol buffers message
453+
instance to serialize.
454+
455+
Returns:
456+
A dict representation of the protocol buffer message.
457+
"""
458+
# For backwards compatibility with protobuf 3.x 4.x
459+
# Remove once support for protobuf 3.x and 4.x is dropped
460+
# https://github.com/googleapis/python-api-core/issues/643
461+
if PROTOBUF_VERSION[0:2] in ["3.", "4."]:
462+
result = json_format.MessageToDict(
463+
message,
464+
preserving_proto_field_name=True,
465+
including_default_value_fields=True, # type: ignore # backward compatibility
466+
)
467+
else:
468+
result = json_format.MessageToDict(
469+
message,
470+
preserving_proto_field_name=True,
471+
always_print_fields_with_no_presence=True,
472+
)
473+
474+
return result
475+
461476
@property
462477
def list_operations(
463478
self,

‎packages/google-api-core/noxfile.py‎

Lines changed: 84 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,9 @@
1515
from __future__ import absolute_import
1616
import os
1717
import pathlib
18+
import re
1819
import shutil
20+
import unittest
1921

2022
# https://github.com/google/importlab/issues/25
2123
import nox # pytype: disable=import-error
@@ -26,6 +28,8 @@
2628
# Black and flake8 clash on the syntax for ignoring flake8's F401 in this file.
2729
BLACK_EXCLUDES = ["--exclude", "^/google/api_core/operations_v1/__init__.py"]
2830

31+
PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]
32+
2933
DEFAULT_PYTHON_VERSION = "3.10"
3034
CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute()
3135

@@ -72,17 +76,46 @@ def blacken(session):
7276
session.run("black", *BLACK_EXCLUDES, *BLACK_PATHS)
7377

7478

75-
def default(session, install_grpc=True):
79+
def install_prerelease_dependencies(session, constraints_path):
80+
with open(constraints_path, encoding="utf-8") as constraints_file:
81+
constraints_text = constraints_file.read()
82+
# Ignore leading whitespace and comment lines.
83+
constraints_deps = [
84+
match.group(1)
85+
for match in re.finditer(
86+
r"^\s*(\S+)(?===\S+)", constraints_text, flags=re.MULTILINE
87+
)
88+
]
89+
session.install(*constraints_deps)
90+
prerel_deps = [
91+
"google-auth",
92+
"googleapis-common-protos",
93+
"grpcio",
94+
"grpcio-status",
95+
"proto-plus",
96+
"protobuf",
97+
]
98+
99+
for dep in prerel_deps:
100+
session.install("--pre", "--no-deps", "--upgrade", dep)
101+
102+
# Remaining dependencies
103+
other_deps = [
104+
"requests",
105+
]
106+
session.install(*other_deps)
107+
108+
109+
def default(session, install_grpc=True, prerelease=False):
76110
"""Default unit test session.
77111
78112
This is intended to be run **without** an interpreter set, so
79113
that the current ``python`` (on the ``PATH``) or the version of
80114
Python corresponding to the ``nox`` binary the ``PATH`` can
81115
run the tests.
82116
"""
83-
constraints_path = str(
84-
CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt"
85-
)
117+
if prerelease and not install_grpc:
118+
unittest.skip("The pre-release session cannot be run without grpc")
86119

87120
session.install(
88121
"dataclasses",
@@ -92,10 +125,36 @@ def default(session, install_grpc=True):
92125
"pytest-xdist",
93126
)
94127

95-
if install_grpc:
96-
session.install("-e", ".[grpc]", "-c", constraints_path)
128+
constraints_dir = str(CURRENT_DIRECTORY / "testing")
129+
130+
if prerelease:
131+
install_prerelease_dependencies(
132+
session, f"{constraints_dir}/constraints-{PYTHON_VERSIONS[0]}.txt"
133+
)
134+
# This *must* be the last install command to get the package from source.
135+
session.install("-e", ".", "--no-deps")
97136
else:
98-
session.install("-e", ".", "-c", constraints_path)
137+
session.install(
138+
"-e",
139+
".[grpc]" if install_grpc else ".",
140+
"-c",
141+
f"{constraints_dir}/constraints-{session.python}.txt",
142+
)
143+
144+
# Print out package versions of dependencies
145+
session.run(
146+
"python", "-c", "import google.protobuf; print(google.protobuf.__version__)"
147+
)
148+
# Support for proto.version was added in v1.23.0
149+
# https://github.com/googleapis/proto-plus-python/releases/tag/v1.23.0
150+
session.run(
151+
"python",
152+
"-c",
153+
"""import proto; hasattr(proto, "version") and print(proto.version.__version__)""",
154+
)
155+
if install_grpc:
156+
session.run("python", "-c", "import grpc; print(grpc.__version__)")
157+
session.run("python", "-c", "import google.auth; print(google.auth.__version__)")
99158

100159
pytest_args = [
101160
"python",
@@ -130,15 +189,26 @@ def default(session, install_grpc=True):
130189
session.run(*pytest_args)
131190

132191

133-
@nox.session(python=["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"])
192+
@nox.session(python=PYTHON_VERSIONS)
134193
def unit(session):
135194
"""Run the unit test suite."""
136195
default(session)
137196

138197

139-
@nox.session(python=["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"])
198+
@nox.session(python=PYTHON_VERSIONS)
199+
def unit_with_prerelease_deps(session):
200+
"""Run the unit test suite."""
201+
default(session, prerelease=True)
202+
203+
204+
@nox.session(python=PYTHON_VERSIONS)
140205
def unit_grpc_gcp(session):
141-
"""Run the unit test suite with grpcio-gcp installed."""
206+
"""
207+
Run the unit test suite with grpcio-gcp installed.
208+
`grpcio-gcp` doesn't support protobuf 4+.
209+
Remove extra `grpcgcp` when protobuf 3.x is dropped.
210+
https://github.com/googleapis/python-api-core/issues/594
211+
"""
142212
constraints_path = str(
143213
CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt"
144214
)
@@ -150,7 +220,7 @@ def unit_grpc_gcp(session):
150220
default(session)
151221

152222

153-
@nox.session(python=["3.8", "3.10", "3.11", "3.12"])
223+
@nox.session(python=PYTHON_VERSIONS)
154224
def unit_wo_grpc(session):
155225
"""Run the unit test suite w/o grpcio installed"""
156226
default(session, install_grpc=False)
@@ -164,10 +234,10 @@ def lint_setup_py(session):
164234
session.run("python", "setup.py", "check", "--restructuredtext", "--strict")
165235

166236

167-
@nox.session(python="3.8")
237+
@nox.session(python=DEFAULT_PYTHON_VERSION)
168238
def pytype(session):
169239
"""Run type-checking."""
170-
session.install(".[grpc]", "pytype >= 2019.3.21")
240+
session.install(".[grpc]", "pytype")
171241
session.run("pytype")
172242

173243

@@ -178,9 +248,7 @@ def mypy(session):
178248
session.install(
179249
"types-setuptools",
180250
"types-requests",
181-
# TODO(https://github.com/googleapis/python-api-core/issues/642):
182-
# Use the latest version of types-protobuf.
183-
"types-protobuf<5",
251+
"types-protobuf",
184252
"types-mock",
185253
"types-dataclasses",
186254
)

‎packages/google-api-core/owlbot.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,8 @@
2929
".flake8", # flake8-import-order, layout
3030
".coveragerc", # layout
3131
"CONTRIBUTING.rst", # no systests
32-
".github/workflows/unittest.yml", # exclude unittest gh action
33-
".github/workflows/lint.yml", # exclude lint gh action
32+
".github/workflows/unittest.yml", # exclude unittest gh action
33+
".github/workflows/lint.yml", # exclude lint gh action
3434
"README.rst",
3535
]
3636
templated_files = common.py_library(microgenerator=True, cov_level=100)

‎packages/google-api-core/pytest.ini‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,3 +19,5 @@ filterwarnings =
1919
ignore:.*pkg_resources is deprecated as an API:DeprecationWarning
2020
# Remove once https://github.com/grpc/grpc/issues/35086 is fixed (and version newer than 1.60.0 is published)
2121
ignore:There is no current event loop:DeprecationWarning
22+
# Remove after support for Python 3.7 is dropped
23+
ignore:After January 1, 2024, new releases of this library will drop support for Python 3.7:DeprecationWarning

‎packages/google-api-core/setup.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@
3030
release_status = "Development Status :: 5 - Production/Stable"
3131
dependencies = [
3232
"googleapis-common-protos >= 1.56.2, < 2.0.dev0",
33-
"protobuf>=3.19.5,<5.0.0.dev0,!=3.20.0,!=3.20.1,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5",
33+
"protobuf>=3.19.5,<6.0.0.dev0,!=3.20.0,!=3.20.1,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5",
3434
"proto-plus >= 1.22.3, <2.0.0dev",
3535
"google-auth >= 2.14.1, < 3.0.dev0",
3636
"requests >= 2.18.0, < 3.0.0.dev0",

0 commit comments

Comments
 (0)