Skip to content

Commit 63366ca

Browse files
busunkim96bshaffer
busunkim96
and
bshaffer
authored
feat: support self-signed JWT flow for service accounts (#774)
See [RFC (internal only)](https://docs.google.com/document/d/1SNCVTmW6Rtr__u-_V7nsT9PhSzjj1z0P9fAD3YUgRoc/edit#) and https://aip.dev/auth/4111. Support the self-signed JWT flow for service accounts by passing `default_scopes` and `default_host` in calls to the auth library and `create_channel`. This depends on features exposed in the following PRs: googleapis/python-api-core#134, googleapis/google-auth-library-python#665. It may be easier to look at https://github.com/googleapis/python-translate/pull/107/files for a diff on a real library. This change is written so that the library is (temporarily) compatible with older `google-api-core` and `google-auth` versions. Because of this it not possible to reach 100% coverage on a single unit test run. `pytest` runs twice in two of the `nox` sessions. Miscellaneous changes: - sprinkled in `__init__.py` files in subdirs of the `test/` directory, as otherwise pytest-cov seems to fail to collect coverage properly in some instances. - new dependency on `packaging` for Version comparison https://pypi.org/project/packaging/ Co-authored-by: Brent Shaffer <betterbrent@google.com>
1 parent 79b4a01 commit 63366ca

File tree

12 files changed

+458
-62
lines changed

12 files changed

+458
-62
lines changed

‎packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/base.py.j2‎

Lines changed: 78 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,12 @@
22

33
{% block content %}
44
import abc
5-
import typing
5+
from typing import Awaitable, Callable, Dict, Optional, Sequence, Union
6+
import packaging.version
67
import pkg_resources
78

89
from google import auth # type: ignore
10+
import google.api_core # type: ignore
911
from google.api_core import exceptions # type: ignore
1012
from google.api_core import gapic_v1 # type: ignore
1113
from google.api_core import retry as retries # type: ignore
@@ -34,6 +36,18 @@ try:
3436
except pkg_resources.DistributionNotFound:
3537
DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo()
3638

39+
try:
40+
# google.auth.__version__ was added in 1.26.0
41+
_GOOGLE_AUTH_VERSION = auth.__version__
42+
except AttributeError:
43+
try: # try pkg_resources if it is available
44+
_GOOGLE_AUTH_VERSION = pkg_resources.get_distribution("google-auth").version
45+
except pkg_resources.DistributionNotFound: # pragma: NO COVER
46+
_GOOGLE_AUTH_VERSION = None
47+
48+
_API_CORE_VERSION = google.api_core.__version__
49+
50+
3751
class {{ service.name }}Transport(abc.ABC):
3852
"""Abstract transport class for {{ service.name }}."""
3953

@@ -43,13 +57,15 @@ class {{ service.name }}Transport(abc.ABC):
4357
{%- endfor %}
4458
)
4559

60+
DEFAULT_HOST: str = {% if service.host %}'{{ service.host }}'{% else %}{{ '' }}{% endif %}
61+
4662
def __init__(
4763
self, *,
48-
host: str{% if service.host %} = '{{ service.host }}'{% endif %},
64+
host: str = DEFAULT_HOST,
4965
credentials: credentials.Credentials = None,
50-
credentials_file: typing.Optional[str] = None,
51-
scopes: typing.Optional[typing.Sequence[str]] = AUTH_SCOPES,
52-
quota_project_id: typing.Optional[str] = None,
66+
credentials_file: Optional[str] = None,
67+
scopes: Optional[Sequence[str]] = None,
68+
quota_project_id: Optional[str] = None,
5369
client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO,
5470
**kwargs,
5571
) -> None:
@@ -66,7 +82,7 @@ class {{ service.name }}Transport(abc.ABC):
6682
credentials_file (Optional[str]): A file with credentials that can
6783
be loaded with :func:`google.auth.load_credentials_from_file`.
6884
This argument is mutually exclusive with credentials.
69-
scope (Optional[Sequence[str]]): A list of scopes.
85+
scopes (Optional[Sequence[str]]): A list of scopes.
7086
quota_project_id (Optional[str]): An optional project to use for billing
7187
and quota.
7288
client_info (google.api_core.gapic_v1.client_info.ClientInfo):
@@ -80,6 +96,8 @@ class {{ service.name }}Transport(abc.ABC):
8096
host += ':443'
8197
self._host = host
8298

99+
scopes_kwargs = self._get_scopes_kwargs(self._host, scopes)
100+
83101
# Save the scopes.
84102
self._scopes = scopes or self.AUTH_SCOPES
85103

@@ -91,17 +109,59 @@ class {{ service.name }}Transport(abc.ABC):
91109
if credentials_file is not None:
92110
credentials, _ = auth.load_credentials_from_file(
93111
credentials_file,
94-
scopes=self._scopes,
112+
**scopes_kwargs,
95113
quota_project_id=quota_project_id
96114
)
97115

98116
elif credentials is None:
99-
credentials, _ = auth.default(scopes=self._scopes, quota_project_id=quota_project_id)
117+
credentials, _ = auth.default(**scopes_kwargs, quota_project_id=quota_project_id)
100118

101119
# Save the credentials.
102120
self._credentials = credentials
103121

104122

123+
# TODO(busunkim): These two class methods are in the base transport
124+
# to avoid duplicating code across the transport classes. These functions
125+
# should be deleted once the minimum required versions of google-api-core
126+
# and google-auth are increased.
127+
128+
# TODO: Remove this function once google-auth >= 1.25.0 is required
129+
@classmethod
130+
def _get_scopes_kwargs(cls, host: str, scopes: Optional[Sequence[str]]) -> Dict[str, Optional[Sequence[str]]]:
131+
"""Returns scopes kwargs to pass to google-auth methods depending on the google-auth version"""
132+
133+
scopes_kwargs = {}
134+
135+
if _GOOGLE_AUTH_VERSION and (
136+
packaging.version.parse(_GOOGLE_AUTH_VERSION)
137+
>= packaging.version.parse("1.25.0")
138+
):
139+
scopes_kwargs = {"scopes": scopes, "default_scopes": cls.AUTH_SCOPES}
140+
else:
141+
scopes_kwargs = {"scopes": scopes or cls.AUTH_SCOPES}
142+
143+
return scopes_kwargs
144+
145+
# TODO: Remove this function once google-api-core >= 1.26.0 is required
146+
@classmethod
147+
def _get_self_signed_jwt_kwargs(cls, host: str, scopes: Optional[Sequence[str]]) -> Dict[str, Union[Optional[Sequence[str]], str]]:
148+
"""Returns kwargs to pass to grpc_helpers.create_channel depending on the google-api-core version"""
149+
150+
self_signed_jwt_kwargs: Dict[str, Union[Optional[Sequence[str]], str]] = {}
151+
152+
if _API_CORE_VERSION and (
153+
packaging.version.parse(_API_CORE_VERSION)
154+
>= packaging.version.parse("1.26.0")
155+
):
156+
self_signed_jwt_kwargs["default_scopes"] = cls.AUTH_SCOPES
157+
self_signed_jwt_kwargs["scopes"] = scopes
158+
self_signed_jwt_kwargs["default_host"] = cls.DEFAULT_HOST
159+
else:
160+
self_signed_jwt_kwargs["scopes"] = scopes or cls.AUTH_SCOPES
161+
162+
return self_signed_jwt_kwargs
163+
164+
105165
def _prep_wrapped_messages(self, client_info):
106166
# Precompute the wrapped methods.
107167
self._wrapped_methods = {
@@ -138,11 +198,11 @@ class {{ service.name }}Transport(abc.ABC):
138198
{%- for method in service.methods.values() %}
139199

140200
@property
141-
def {{ method.name|snake_case }}(self) -> typing.Callable[
201+
def {{ method.name|snake_case }}(self) -> Callable[
142202
[{{ method.input.ident }}],
143-
typing.Union[
203+
Union[
144204
{{ method.output.ident }},
145-
typing.Awaitable[{{ method.output.ident }}]
205+
Awaitable[{{ method.output.ident }}]
146206
]]:
147207
raise NotImplementedError()
148208
{%- endfor %}
@@ -152,29 +212,29 @@ class {{ service.name }}Transport(abc.ABC):
152212
@property
153213
def set_iam_policy(
154214
self,
155-
) -> typing.Callable[
215+
) -> Callable[
156216
[iam_policy.SetIamPolicyRequest],
157-
typing.Union[policy.Policy, typing.Awaitable[policy.Policy]],
217+
Union[policy.Policy, Awaitable[policy.Policy]],
158218
]:
159219
raise NotImplementedError()
160220

161221
@property
162222
def get_iam_policy(
163223
self,
164-
) -> typing.Callable[
224+
) -> Callable[
165225
[iam_policy.GetIamPolicyRequest],
166-
typing.Union[policy.Policy, typing.Awaitable[policy.Policy]],
226+
Union[policy.Policy, Awaitable[policy.Policy]],
167227
]:
168228
raise NotImplementedError()
169229

170230
@property
171231
def test_iam_permissions(
172232
self,
173-
) -> typing.Callable[
233+
) -> Callable[
174234
[iam_policy.TestIamPermissionsRequest],
175-
typing.Union[
235+
Union[
176236
iam_policy.TestIamPermissionsResponse,
177-
typing.Awaitable[iam_policy.TestIamPermissionsResponse],
237+
Awaitable[iam_policy.TestIamPermissionsResponse],
178238
],
179239
]:
180240
raise NotImplementedError()

‎packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/grpc.py.j2‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
{% block content %}
44
import warnings
5-
from typing import Callable, Dict, Optional, Sequence, Tuple
5+
from typing import Callable, Dict, Optional, Sequence, Tuple, Union
66

77
from google.api_core import grpc_helpers # type: ignore
88
{%- if service.has_lro %}
@@ -202,13 +202,15 @@ class {{ service.name }}GrpcTransport({{ service.name }}Transport):
202202
google.api_core.exceptions.DuplicateCredentialArgs: If both ``credentials``
203203
and ``credentials_file`` are passed.
204204
"""
205-
scopes = scopes or cls.AUTH_SCOPES
205+
206+
self_signed_jwt_kwargs = cls._get_self_signed_jwt_kwargs(host, scopes)
207+
206208
return grpc_helpers.create_channel(
207209
host,
208210
credentials=credentials,
209211
credentials_file=credentials_file,
210-
scopes=scopes,
211212
quota_project_id=quota_project_id,
213+
**self_signed_jwt_kwargs,
212214
**kwargs
213215
)
214216

‎packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/grpc_asyncio.py.j2‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
{% block content %}
44
import warnings
5-
from typing import Awaitable, Callable, Dict, Optional, Sequence, Tuple
5+
from typing import Awaitable, Callable, Dict, Optional, Sequence, Tuple, Union
66

77
from google.api_core import gapic_v1 # type: ignore
88
from google.api_core import grpc_helpers_async # type: ignore
@@ -12,6 +12,7 @@ from google.api_core import operations_v1 # type: ignore
1212
from google import auth # type: ignore
1313
from google.auth import credentials # type: ignore
1414
from google.auth.transport.grpc import SslCredentials # type: ignore
15+
import packaging.version
1516

1617
import grpc # type: ignore
1718
from grpc.experimental import aio # type: ignore
@@ -75,13 +76,15 @@ class {{ service.grpc_asyncio_transport_name }}({{ service.name }}Transport):
7576
Returns:
7677
aio.Channel: A gRPC AsyncIO channel object.
7778
"""
78-
scopes = scopes or cls.AUTH_SCOPES
79+
80+
self_signed_jwt_kwargs = cls._get_self_signed_jwt_kwargs(host, scopes)
81+
7982
return grpc_helpers_async.create_channel(
8083
host,
8184
credentials=credentials,
8285
credentials_file=credentials_file,
83-
scopes=scopes,
8486
quota_project_id=quota_project_id,
87+
**self_signed_jwt_kwargs,
8588
**kwargs
8689
)
8790

@@ -163,7 +166,6 @@ class {{ service.grpc_asyncio_transport_name }}({{ service.name }}Transport):
163166
# If a channel was explicitly provided, set it.
164167
self._grpc_channel = channel
165168
self._ssl_channel_credentials = None
166-
167169
else:
168170
if api_mtls_endpoint:
169171
host = api_mtls_endpoint

‎packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/rest.py.j2‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -81,12 +81,14 @@ class {{ service.name }}RestTransport({{ service.name }}Transport):
8181
"""
8282
# Run the base constructor
8383
# TODO(yon-mg): resolve other ctor params i.e. scopes, quota, etc.
84+
# TODO: When custom host (api_endpoint) is set, `scopes` must *also* be set on the
85+
# credentials object
8486
super().__init__(
8587
host=host,
8688
credentials=credentials,
8789
client_info=client_info,
8890
)
89-
self._session = AuthorizedSession(self._credentials)
91+
self._session = AuthorizedSession(self._credentials, default_host=self.DEFAULT_HOST)
9092
{%- if service.has_lro %}
9193
self._operations_client = None
9294
{%- endif %}
@@ -106,11 +108,14 @@ class {{ service.name }}RestTransport({{ service.name }}Transport):
106108
# Sanity check: Only create a new client if we do not already have one.
107109
if self._operations_client is None:
108110
from google.api_core import grpc_helpers
111+
112+
self_signed_jwt_kwargs = cls._get_self_signed_jwt_kwargs(self._host, self._scopes)
113+
109114
self._operations_client = operations_v1.OperationsClient(
110115
grpc_helpers.create_channel(
111116
self._host,
112117
credentials=self._credentials,
113-
scopes=self.AUTH_SCOPES,
118+
**self_signed_jwt_kwargs,
114119
options=[
115120
("grpc.max_send_message_length", -1),
116121
("grpc.max_receive_message_length", -1),

‎packages/gapic-generator/gapic/templates/.coveragerc.j2‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@
22
branch = True
33

44
[report]
5-
fail_under = 100
65
show_missing = True
76
omit =
87
{{ api.naming.module_namespace|join("/") }}/{{ api.naming.module_name }}/__init__.py

‎packages/gapic-generator/gapic/templates/noxfile.py.j2‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,28 @@
22

33
{% block content %}
44
import os
5+
import pathlib
56
import shutil
7+
import subprocess
8+
import sys
9+
610

711
import nox # type: ignore
812

13+
CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute()
14+
15+
LOWER_BOUND_CONSTRAINTS_FILE = CURRENT_DIRECTORY / "constraints.txt"
16+
PACKAGE_NAME = subprocess.check_output([sys.executable, "setup.py", "--name"], encoding="utf-8")
17+
18+
19+
nox.sessions = [
20+
"unit",
21+
"cover",
22+
"mypy",
23+
"check_lower_bounds"
24+
# exclude update_lower_bounds from default
25+
"docs",
26+
]
927

1028
@nox.session(python=['3.6', '3.7', '3.8', '3.9'])
1129
def unit(session):
@@ -25,6 +43,18 @@ def unit(session):
2543
)
2644

2745

46+
@nox.session(python='3.7')
47+
def cover(session):
48+
"""Run the final coverage report.
49+
This outputs the coverage report aggregating coverage from the unit
50+
test runs (not system test runs), and then erases coverage data.
51+
"""
52+
session.install("coverage", "pytest-cov")
53+
session.run("coverage", "report", "--show-missing", "--fail-under=100")
54+
55+
session.run("coverage", "erase")
56+
57+
2858
@nox.session(python=['3.6', '3.7'])
2959
def mypy(session):
3060
"""Run the type checker."""
@@ -40,6 +70,38 @@ def mypy(session):
4070
{%- endif %}
4171
)
4272

73+
74+
@nox.session
75+
def update_lower_bounds(session):
76+
"""Update lower bounds in constraints.txt to match setup.py"""
77+
session.install('google-cloud-testutils')
78+
session.install('.')
79+
80+
session.run(
81+
'lower-bound-checker',
82+
'update',
83+
'--package-name',
84+
PACKAGE_NAME,
85+
'--constraints-file',
86+
str(LOWER_BOUND_CONSTRAINTS_FILE),
87+
)
88+
89+
90+
@nox.session
91+
def check_lower_bounds(session):
92+
"""Check lower bounds in setup.py are reflected in constraints file"""
93+
session.install('google-cloud-testutils')
94+
session.install('.')
95+
96+
session.run(
97+
'lower-bound-checker',
98+
'check',
99+
'--package-name',
100+
PACKAGE_NAME,
101+
'--constraints-file',
102+
str(LOWER_BOUND_CONSTRAINTS_FILE),
103+
)
104+
43105
@nox.session(python='3.6')
44106
def docs(session):
45107
"""Build the docs for this library."""

‎packages/gapic-generator/gapic/templates/setup.py.j2‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,9 @@ setuptools.setup(
2929
'google-api-core[grpc] >= 1.22.2, < 2.0.0dev',
3030
'libcst >= 0.2.5',
3131
'proto-plus >= 1.15.0',
32+
'packaging >= 14.3',
3233
{%- if api.requires_package(('google', 'iam', 'v1')) or opts.add_iam_methods %}
33-
'grpc-google-iam-v1',
34+
'grpc-google-iam-v1 >= 0.12.3, < 0.13dev',
3435
{%- endif %}
3536
),
3637
python_requires='>=3.6',

‎packages/gapic-generator/gapic/templates/tests/__init__.py.j2‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
2+
{% extends '_base.py.j2' %}

0 commit comments

Comments
 (0)