Skip to content
This repository was archived by the owner on Mar 6, 2026. It is now read-only.

test: adds pytest-xdist to speed up processing of CI/CD checks #2153

Merged
chalmerlowe merged 8 commits into from
Apr 11, 2025
Merged

test: adds pytest-xdist to speed up processing of CI/CD checks #2153

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
d083549
experimentation using pytest-xdist
chalmerlowe Apr 2, 2025
ff2129e
adds pytest-xdist to nox system session for experimentation
chalmerlowe Apr 3, 2025
5e29869
adds pytest-xdist install AND -n=auto argument
chalmerlowe Apr 8, 2025
fc87d78
updates sample noxfiles
chalmerlowe Apr 11, 2025
f61f3ce
Merge branch 'main' into test-xdist-experimentation
chalmerlowe Apr 11, 2025
af53167
updates pytest version in requirements-test.txt files
chalmerlowe Apr 11, 2025
8c0604f
Update samples/notebooks/requirements-test.txt
chalmerlowe Apr 11, 2025
fccc84e
Update samples/notebooks/requirements-test.txt
chalmerlowe Apr 11, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 20 additions & 3 deletions noxfile.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,7 @@ def default(session, install_extras=True):
"pytest",
"google-cloud-testutils",
"pytest-cov",
"pytest-xdist",
"freezegun",
"-c",
constraints_path,
Expand Down Expand Up @@ -129,6 +130,7 @@ def default(session, install_extras=True):
# Run py.test against the unit tests.
session.run(
"py.test",
"-n=auto",
Copy link
Copy Markdown
Contributor

@tswast tswast Apr 11, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think the default is the number of CPUs on the instance, right? I recall in bigframes we set this a bit higher than that and it seemed to help.

Copy link
Copy Markdown
Collaborator Author

@chalmerlowe chalmerlowe Apr 11, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I believe you are correct, that the default is the number of CPUs on the instance.

"--quiet",
"-W default::PendingDeprecationWarning",
"--cov=google/cloud/bigquery",
Expand Down Expand Up @@ -224,7 +226,12 @@ def system(session):

# Install all test dependencies, then install local packages in place.
session.install(
"pytest", "psutil", "google-cloud-testutils", "-c", constraints_path
"pytest",
"psutil",
"pytest-xdist",
"google-cloud-testutils",
"-c",
constraints_path,
)
if os.environ.get("GOOGLE_API_USE_CLIENT_CERTIFICATE", "") == "true":
# mTLS test requires pyopenssl and latest google-cloud-storage
Expand Down Expand Up @@ -257,6 +264,7 @@ def system(session):
# Run py.test against the system tests.
session.run(
"py.test",
"-n=auto",
"--quiet",
"-W default::PendingDeprecationWarning",
os.path.join("tests", "system"),
Expand Down Expand Up @@ -310,7 +318,9 @@ def snippets(session):
)

# Install all test dependencies, then install local packages in place.
session.install("pytest", "google-cloud-testutils", "-c", constraints_path)
session.install(
"pytest", "pytest-xdist", "google-cloud-testutils", "-c", constraints_path
)
session.install("google-cloud-storage", "-c", constraints_path)
session.install("grpcio", "-c", constraints_path)

Expand All @@ -326,9 +336,12 @@ def snippets(session):
# Run py.test against the snippets tests.
# Skip tests in samples/snippets, as those are run in a different session
# using the nox config from that directory.
session.run("py.test", os.path.join("docs", "snippets.py"), *session.posargs)
session.run(
"py.test", "-n=auto", os.path.join("docs", "snippets.py"), *session.posargs
)
session.run(
"py.test",
"-n=auto",
"samples",
"-W default::PendingDeprecationWarning",
"--ignore=samples/desktopapp",
Expand Down Expand Up @@ -393,6 +406,7 @@ def prerelease_deps(session):
"google-cloud-testutils",
"psutil",
"pytest",
"pytest-xdist",
"pytest-cov",
)

Expand Down Expand Up @@ -439,18 +453,21 @@ def prerelease_deps(session):
# Run all tests, except a few samples tests which require extra dependencies.
session.run(
"py.test",
"-n=auto",
"tests/unit",
"-W default::PendingDeprecationWarning",
)

session.run(
"py.test",
"-n=auto",
"tests/system",
"-W default::PendingDeprecationWarning",
)

session.run(
"py.test",
"-n=auto",
"samples/tests",
"-W default::PendingDeprecationWarning",
)
Expand Down
1 change: 1 addition & 0 deletions samples/desktopapp/requirements-test.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
google-cloud-testutils==1.6.0
pytest==8.3.5
mock==5.2.0
pytest-xdist==3.6.1
1 change: 1 addition & 0 deletions samples/geography/requirements-test.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
pytest==8.3.5
mock==5.2.0
pytest-xdist==3.6.1
1 change: 1 addition & 0 deletions samples/magics/requirements-test.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
google-cloud-testutils==1.6.0
pytest==8.3.5
mock==5.2.0
pytest-xdist==3.6.1
1 change: 1 addition & 0 deletions samples/notebooks/requirements-test.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
google-cloud-testutils==1.6.0
pytest==8.3.5
mock==5.2.0
pytest-xdist==3.6.1
1 change: 1 addition & 0 deletions samples/snippets/requirements-test.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,3 +2,4 @@
google-cloud-testutils==1.6.0
pytest==8.3.5
mock==5.2.0
pytest-xdist==3.6.1