Skip to content
This repository was archived by the owner on Apr 1, 2026. It is now read-only.

Commit 58fa738

Browse files
tseaverpartheagcf-owl-bot[bot]
authored
ci: run samples under Python 3.9 / 3.10 (#478)
* ci: run samples under Python 3.9 / 3.10 Refresh each sample's noxfile via: ----------------------------- %< ----------------------------- $ for noxfile in samples/*/noxfile.py; do echo "Refreshing $noxfile"; wget -O $noxfile https://github.com/GoogleCloudPlatform/python-docs-samples/raw/main/noxfile-template.py echo "Blackening samples for $noxfile" nox -f $noxfile -s blacken done ----------------------------- %< ----------------------------- Closes #477. * fix: disable install-from-sorce for beam sample Per #203. * fix: skip beam sample for Python 3.10 Beam-related wheels are not yet available. * fix: also refresh noxfiles for 'samples/snippets' * ci: don't enforce type hints on old samples * resolve issue where samples templates are not updated * 🦉 Updates from OwlBot See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * resolve mypy error Name __path__ already defined * add workaroud from PR #203 Co-authored-by: Anthonios Partheniou <partheniou@google.com> Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>
1 parent fec06fc commit 58fa738

38 files changed

Lines changed: 1395 additions & 843 deletions

‎google/__init__.py‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,6 @@
1-
from typing import List
2-
31
try:
42
import pkg_resources
53

64
pkg_resources.declare_namespace(__name__)
75
except ImportError:
8-
import pkgutil
9-
10-
__path__: List[str] = pkgutil.extend_path(__path__, __name__)
6+
pass

‎google/cloud/__init__.py‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,6 @@
1-
from typing import List
2-
31
try:
42
import pkg_resources
53

64
pkg_resources.declare_namespace(__name__)
75
except ImportError:
8-
import pkgutil
9-
10-
__path__: List[str] = pkgutil.extend_path(__path__, __name__)
6+
pass

‎owlbot.py‎

Lines changed: 9 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -217,20 +217,15 @@ def lint_setup_py(session):
217217
# Samples templates
218218
# ----------------------------------------------------------------------------
219219

220-
sample_files = common.py_samples(samples=True)
221-
for path in sample_files:
222-
s.move(path)
223-
224-
# Note: python-docs-samples is not yet using 'main':
225-
#s.replace(
226-
# "samples/**/*.md",
227-
# r"python-docs-samples/blob/master/",
228-
# "python-docs-samples/blob/main/",
229-
#)
220+
python.py_samples(skip_readmes=True)
221+
230222
s.replace(
231-
"samples/**/*.md",
232-
r"google-cloud-python/blob/master/",
233-
"google-cloud-python/blob/main/",
234-
)
223+
"samples/beam/noxfile.py",
224+
"""INSTALL_LIBRARY_FROM_SOURCE \= os.environ.get\("INSTALL_LIBRARY_FROM_SOURCE", False\) in \(
225+
"True",
226+
"true",
227+
\)""",
228+
"""# todo(kolea2): temporary workaround to install pinned dep version
229+
INSTALL_LIBRARY_FROM_SOURCE = False""")
235230

236231
s.shell.run(["nox", "-s", "blacken"], hide_output=False)

‎samples/beam/hello_world_write.py‎

Lines changed: 21 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -23,42 +23,44 @@ class BigtableOptions(PipelineOptions):
2323
@classmethod
2424
def _add_argparse_args(cls, parser):
2525
parser.add_argument(
26-
'--bigtable-project',
27-
help='The Bigtable project ID, this can be different than your '
28-
'Dataflow project',
29-
default='bigtable-project')
26+
"--bigtable-project",
27+
help="The Bigtable project ID, this can be different than your "
28+
"Dataflow project",
29+
default="bigtable-project",
30+
)
3031
parser.add_argument(
31-
'--bigtable-instance',
32-
help='The Bigtable instance ID',
33-
default='bigtable-instance')
32+
"--bigtable-instance",
33+
help="The Bigtable instance ID",
34+
default="bigtable-instance",
35+
)
3436
parser.add_argument(
35-
'--bigtable-table',
36-
help='The Bigtable table ID in the instance.',
37-
default='bigtable-table')
37+
"--bigtable-table",
38+
help="The Bigtable table ID in the instance.",
39+
default="bigtable-table",
40+
)
3841

3942

4043
class CreateRowFn(beam.DoFn):
4144
def process(self, key):
4245
direct_row = row.DirectRow(row_key=key)
4346
direct_row.set_cell(
44-
"stats_summary",
45-
b"os_build",
46-
b"android",
47-
datetime.datetime.now())
47+
"stats_summary", b"os_build", b"android", datetime.datetime.now()
48+
)
4849
return [direct_row]
4950

5051

5152
def run(argv=None):
5253
"""Build and run the pipeline."""
5354
options = BigtableOptions(argv)
5455
with beam.Pipeline(options=options) as p:
55-
p | beam.Create(["phone#4c410523#20190501",
56-
"phone#4c410523#20190502"]) | beam.ParDo(
57-
CreateRowFn()) | WriteToBigTable(
56+
p | beam.Create(
57+
["phone#4c410523#20190501", "phone#4c410523#20190502"]
58+
) | beam.ParDo(CreateRowFn()) | WriteToBigTable(
5859
project_id=options.bigtable_project,
5960
instance_id=options.bigtable_instance,
60-
table_id=options.bigtable_table)
61+
table_id=options.bigtable_table,
62+
)
6163

6264

63-
if __name__ == '__main__':
65+
if __name__ == "__main__":
6466
run()

‎samples/beam/hello_world_write_test.py‎

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,9 @@
1919

2020
import hello_world_write
2121

22-
PROJECT = os.environ['GOOGLE_CLOUD_PROJECT']
23-
BIGTABLE_INSTANCE = os.environ['BIGTABLE_INSTANCE']
24-
TABLE_ID_PREFIX = 'mobile-time-series-{}'
22+
PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"]
23+
BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"]
24+
TABLE_ID_PREFIX = "mobile-time-series-{}"
2525

2626

2727
@pytest.fixture(scope="module", autouse=True)
@@ -34,17 +34,20 @@ def table_id():
3434
if table.exists():
3535
table.delete()
3636

37-
table.create(column_families={'stats_summary': None})
37+
table.create(column_families={"stats_summary": None})
3838
yield table_id
3939

4040
table.delete()
4141

4242

4343
def test_hello_world_write(table_id):
44-
hello_world_write.run([
45-
'--bigtable-project=%s' % PROJECT,
46-
'--bigtable-instance=%s' % BIGTABLE_INSTANCE,
47-
'--bigtable-table=%s' % table_id])
44+
hello_world_write.run(
45+
[
46+
"--bigtable-project=%s" % PROJECT,
47+
"--bigtable-instance=%s" % BIGTABLE_INSTANCE,
48+
"--bigtable-table=%s" % table_id,
49+
]
50+
)
4851

4952
client = bigtable.Client(project=PROJECT, admin=True)
5053
instance = client.instance(BIGTABLE_INSTANCE)

‎samples/beam/noxfile.py‎

Lines changed: 71 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
import os
1818
from pathlib import Path
1919
import sys
20+
from typing import Callable, Dict, List, Optional
2021

2122
import nox
2223

@@ -27,8 +28,9 @@
2728
# WARNING - WARNING - WARNING - WARNING - WARNING
2829
# WARNING - WARNING - WARNING - WARNING - WARNING
2930

30-
# Copy `noxfile_config.py` to your directory and modify it instead.
31+
BLACK_VERSION = "black==19.10b0"
3132

33+
# Copy `noxfile_config.py` to your directory and modify it instead.
3234

3335
# `TEST_CONFIG` dict is a configuration hook that allows users to
3436
# modify the test configurations. The values here should be in sync
@@ -37,24 +39,29 @@
3739

3840
TEST_CONFIG = {
3941
# You can opt out from the test for specific Python versions.
40-
'ignored_versions': ["2.7"],
41-
42+
"ignored_versions": [],
43+
# Old samples are opted out of enforcing Python type hints
44+
# All new samples should feature them
45+
"enforce_type_hints": False,
4246
# An envvar key for determining the project id to use. Change it
4347
# to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a
4448
# build specific Cloud project. You can also use your own string
4549
# to use your own Cloud project.
46-
'gcloud_project_env': 'GOOGLE_CLOUD_PROJECT',
50+
"gcloud_project_env": "GOOGLE_CLOUD_PROJECT",
4751
# 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT',
48-
52+
# If you need to use a specific version of pip,
53+
# change pip_version_override to the string representation
54+
# of the version number, for example, "20.2.4"
55+
"pip_version_override": None,
4956
# A dictionary you want to inject into your test. Don't put any
5057
# secrets here. These values will override predefined values.
51-
'envs': {},
58+
"envs": {},
5259
}
5360

5461

5562
try:
5663
# Ensure we can import noxfile_config in the project's directory.
57-
sys.path.append('.')
64+
sys.path.append(".")
5865
from noxfile_config import TEST_CONFIG_OVERRIDE
5966
except ImportError as e:
6067
print("No user noxfile_config found: detail: {}".format(e))
@@ -64,37 +71,41 @@
6471
TEST_CONFIG.update(TEST_CONFIG_OVERRIDE)
6572

6673

67-
def get_pytest_env_vars():
74+
def get_pytest_env_vars() -> Dict[str, str]:
6875
"""Returns a dict for pytest invocation."""
6976
ret = {}
7077

7178
# Override the GCLOUD_PROJECT and the alias.
72-
env_key = TEST_CONFIG['gcloud_project_env']
79+
env_key = TEST_CONFIG["gcloud_project_env"]
7380
# This should error out if not set.
74-
ret['GOOGLE_CLOUD_PROJECT'] = os.environ[env_key]
81+
ret["GOOGLE_CLOUD_PROJECT"] = os.environ[env_key]
7582

7683
# Apply user supplied envs.
77-
ret.update(TEST_CONFIG['envs'])
84+
ret.update(TEST_CONFIG["envs"])
7885
return ret
7986

8087

8188
# DO NOT EDIT - automatically generated.
82-
# All versions used to tested samples.
83-
ALL_VERSIONS = ["2.7", "3.6", "3.7", "3.8"]
89+
# All versions used to test samples.
90+
ALL_VERSIONS = ["3.6", "3.7", "3.8", "3.9", "3.10"]
8491

8592
# Any default versions that should be ignored.
86-
IGNORED_VERSIONS = TEST_CONFIG['ignored_versions']
93+
IGNORED_VERSIONS = TEST_CONFIG["ignored_versions"]
8794

8895
TESTED_VERSIONS = sorted([v for v in ALL_VERSIONS if v not in IGNORED_VERSIONS])
8996

9097
# todo(kolea2): temporary workaround to install pinned dep version
9198
INSTALL_LIBRARY_FROM_SOURCE = False
99+
100+
# Error if a python version is missing
101+
nox.options.error_on_missing_interpreters = True
102+
92103
#
93104
# Style Checks
94105
#
95106

96107

97-
def _determine_local_import_names(start_dir):
108+
def _determine_local_import_names(start_dir: str) -> List[str]:
98109
"""Determines all import names that should be considered "local".
99110
100111
This is used when running the linter to insure that import order is
@@ -132,18 +143,34 @@ def _determine_local_import_names(start_dir):
132143

133144

134145
@nox.session
135-
def lint(session):
136-
session.install("flake8", "flake8-import-order")
146+
def lint(session: nox.sessions.Session) -> None:
147+
if not TEST_CONFIG["enforce_type_hints"]:
148+
session.install("flake8", "flake8-import-order")
149+
else:
150+
session.install("flake8", "flake8-import-order", "flake8-annotations")
137151

138152
local_names = _determine_local_import_names(".")
139153
args = FLAKE8_COMMON_ARGS + [
140154
"--application-import-names",
141155
",".join(local_names),
142-
"."
156+
".",
143157
]
144158
session.run("flake8", *args)
145159

146160

161+
#
162+
# Black
163+
#
164+
165+
166+
@nox.session
167+
def blacken(session: nox.sessions.Session) -> None:
168+
session.install(BLACK_VERSION)
169+
python_files = [path for path in os.listdir(".") if path.endswith(".py")]
170+
171+
session.run("black", *python_files)
172+
173+
147174
#
148175
# Sample Tests
149176
#
@@ -152,13 +179,24 @@ def lint(session):
152179
PYTEST_COMMON_ARGS = ["--junitxml=sponge_log.xml"]
153180

154181

155-
def _session_tests(session, post_install=None):
182+
def _session_tests(
183+
session: nox.sessions.Session, post_install: Callable = None
184+
) -> None:
185+
if TEST_CONFIG["pip_version_override"]:
186+
pip_version = TEST_CONFIG["pip_version_override"]
187+
session.install(f"pip=={pip_version}")
156188
"""Runs py.test for a particular project."""
157189
if os.path.exists("requirements.txt"):
158-
session.install("-r", "requirements.txt")
190+
if os.path.exists("constraints.txt"):
191+
session.install("-r", "requirements.txt", "-c", "constraints.txt")
192+
else:
193+
session.install("-r", "requirements.txt")
159194

160195
if os.path.exists("requirements-test.txt"):
161-
session.install("-r", "requirements-test.txt")
196+
if os.path.exists("constraints-test.txt"):
197+
session.install("-r", "requirements-test.txt", "-c", "constraints-test.txt")
198+
else:
199+
session.install("-r", "requirements-test.txt")
162200

163201
if INSTALL_LIBRARY_FROM_SOURCE:
164202
session.install("-e", _get_repo_root())
@@ -173,27 +211,27 @@ def _session_tests(session, post_install=None):
173211
# on travis where slow and flaky tests are excluded.
174212
# See http://doc.pytest.org/en/latest/_modules/_pytest/main.html
175213
success_codes=[0, 5],
176-
env=get_pytest_env_vars()
214+
env=get_pytest_env_vars(),
177215
)
178216

179217

180218
@nox.session(python=ALL_VERSIONS)
181-
def py(session):
219+
def py(session: nox.sessions.Session) -> None:
182220
"""Runs py.test for a sample using the specified version of Python."""
183221
if session.python in TESTED_VERSIONS:
184222
_session_tests(session)
185223
else:
186-
session.skip("SKIPPED: {} tests are disabled for this sample.".format(
187-
session.python
188-
))
224+
session.skip(
225+
"SKIPPED: {} tests are disabled for this sample.".format(session.python)
226+
)
189227

190228

191229
#
192230
# Readmegen
193231
#
194232

195233

196-
def _get_repo_root():
234+
def _get_repo_root() -> Optional[str]:
197235
""" Returns the root folder of the project. """
198236
# Get root of this repository. Assume we don't have directories nested deeper than 10 items.
199237
p = Path(os.getcwd())
@@ -202,6 +240,11 @@ def _get_repo_root():
202240
break
203241
if Path(p / ".git").exists():
204242
return str(p)
243+
# .git is not available in repos cloned via Cloud Build
244+
# setup.py is always in the library's root, so use that instead
245+
# https://github.com/googleapis/synthtool/issues/792
246+
if Path(p / "setup.py").exists():
247+
return str(p)
205248
p = p.parent
206249
raise Exception("Unable to detect repository root.")
207250

@@ -211,7 +254,7 @@ def _get_repo_root():
211254

212255
@nox.session
213256
@nox.parametrize("path", GENERATED_READMES)
214-
def readmegen(session, path):
257+
def readmegen(session: nox.sessions.Session, path: str) -> None:
215258
"""(Re-)generates the readme for a sample."""
216259
session.install("jinja2", "pyyaml")
217260
dir_ = os.path.dirname(path)

0 commit comments

Comments
 (0)