diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 000000000..9372faac2 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,39 @@ +on: + pull_request: + branches: + - main +name: docs +jobs: + docs: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + - name: Install nox + run: | + python -m pip install --upgrade setuptools pip wheel + python -m pip install nox + - name: Run docs session + run: | + nox -s docs-3.10 + + docfx: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + - name: Install nox + run: | + python -m pip install --upgrade setuptools pip wheel + python -m pip install nox + - name: Run docfx session + run: | + nox -s docfx-3.10 diff --git a/.kokoro/presubmit/presubmit.cfg b/.kokoro/presubmit/presubmit.cfg deleted file mode 100644 index ac4cc5847..000000000 --- a/.kokoro/presubmit/presubmit.cfg +++ /dev/null @@ -1,7 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -# Disable system tests. -env_vars: { - key: "NOX_SESSION" - value: "unit_noextras unit cover docs docfx" -} diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f7166d44..3b29a6a41 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,24 @@ [1]: https://pypi.org/project/google-cloud-bigquery/#history +## [3.34.0](https://github.com/googleapis/python-bigquery/compare/v3.33.0...v3.34.0) (2025-05-27) + + +### Features + +* Job creation mode GA ([#2190](https://github.com/googleapis/python-bigquery/issues/2190)) ([64cd39f](https://github.com/googleapis/python-bigquery/commit/64cd39fb395c4a03ef6d2ec8261e1709477b2186)) + + +### Bug Fixes + +* **deps:** Update all dependencies ([#2184](https://github.com/googleapis/python-bigquery/issues/2184)) ([12490f2](https://github.com/googleapis/python-bigquery/commit/12490f2f03681516465fc34217dcdf57000f6fdd)) + + +### Documentation + +* Update query.py ([#2192](https://github.com/googleapis/python-bigquery/issues/2192)) ([9b5ee78](https://github.com/googleapis/python-bigquery/commit/9b5ee78f046d9ca3f758eeca6244b8485fe35875)) +* Use query_and_wait in the array parameters sample ([#2202](https://github.com/googleapis/python-bigquery/issues/2202)) ([28a9994](https://github.com/googleapis/python-bigquery/commit/28a9994792ec90a6a4d16835faf2137c09c0fb02)) + ## [3.33.0](https://github.com/googleapis/python-bigquery/compare/v3.32.0...v3.33.0) (2025-05-19) diff --git a/google/cloud/bigquery/_helpers.py b/google/cloud/bigquery/_helpers.py index 76c4f1fbd..c7d7705e0 100644 --- a/google/cloud/bigquery/_helpers.py +++ b/google/cloud/bigquery/_helpers.py @@ -388,7 +388,7 @@ def range_to_py(self, value, field): class DataFrameCellDataParser(CellDataParser): - """Override of CellDataParser to handle differences in expection of values in DataFrame-like outputs. + """Override of CellDataParser to handle differences in expression of values in DataFrame-like outputs. This is used to turn the output of the REST API into a pyarrow Table, emulating the serialized arrow from the BigQuery Storage Read API. diff --git a/google/cloud/bigquery/_job_helpers.py b/google/cloud/bigquery/_job_helpers.py index 4a884ada5..888dc1e73 100644 --- a/google/cloud/bigquery/_job_helpers.py +++ b/google/cloud/bigquery/_job_helpers.py @@ -37,7 +37,6 @@ import copy import functools -import os import uuid import textwrap from typing import Any, Dict, Optional, TYPE_CHECKING, Union @@ -400,12 +399,6 @@ def query_and_wait( ) -> table.RowIterator: """Run the query, wait for it to finish, and return the results. - While ``jobCreationMode=JOB_CREATION_OPTIONAL`` is in preview in the - ``jobs.query`` REST API, use the default ``jobCreationMode`` unless - the environment variable ``QUERY_PREVIEW_ENABLED=true``. After - ``jobCreationMode`` is GA, this method will always use - ``jobCreationMode=JOB_CREATION_OPTIONAL``. See: - https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs/query Args: client: @@ -500,9 +493,8 @@ def query_and_wait( request_body["maxResults"] = min(page_size, max_results) elif page_size is not None or max_results is not None: request_body["maxResults"] = page_size or max_results - - if os.getenv("QUERY_PREVIEW_ENABLED", "").casefold() == "true": - request_body["jobCreationMode"] = "JOB_CREATION_OPTIONAL" + if client.default_job_creation_mode: + request_body["jobCreationMode"] = client.default_job_creation_mode def do_query(): request_body["requestId"] = make_job_id() diff --git a/google/cloud/bigquery/_pandas_helpers.py b/google/cloud/bigquery/_pandas_helpers.py index 6691e7ef6..10a5c59bb 100644 --- a/google/cloud/bigquery/_pandas_helpers.py +++ b/google/cloud/bigquery/_pandas_helpers.py @@ -1144,7 +1144,7 @@ def determine_requested_streams( """ if preserve_order: - # If preserve order is set, it takes precendence. + # If preserve order is set, it takes precedence. # Limit the requested streams to 1, to ensure that order # is preserved) return 1 diff --git a/google/cloud/bigquery/client.py b/google/cloud/bigquery/client.py index 8ad1586f4..c6873545b 100644 --- a/google/cloud/bigquery/client.py +++ b/google/cloud/bigquery/client.py @@ -221,6 +221,10 @@ class Client(ClientWithProject): client_options (Optional[Union[google.api_core.client_options.ClientOptions, Dict]]): Client options used to set user options on the client. API Endpoint should be set through client_options. + default_job_creation_mode (Optional[str]): + Sets the default job creation mode used by query methods such as + query_and_wait(). For lightweight queries, JOB_CREATION_OPTIONAL is + generally recommended. Raises: google.auth.exceptions.DefaultCredentialsError: @@ -243,6 +247,7 @@ def __init__( client_options: Optional[ Union[google.api_core.client_options.ClientOptions, Dict[str, Any]] ] = None, + default_job_creation_mode: Optional[str] = None, ) -> None: if client_options is None: client_options = {} @@ -277,6 +282,7 @@ def __init__( self._connection = Connection(self, **kw_args) self._location = location self._default_load_job_config = copy.deepcopy(default_load_job_config) + self.default_job_creation_mode = default_job_creation_mode # Use property setter so validation can run. self.default_query_job_config = default_query_job_config @@ -286,6 +292,15 @@ def location(self): """Default location for jobs / datasets / tables.""" return self._location + @property + def default_job_creation_mode(self): + """Default job creation mode used for query execution.""" + return self._default_job_creation_mode + + @default_job_creation_mode.setter + def default_job_creation_mode(self, value: Optional[str]): + self._default_job_creation_mode = value + @property def default_query_job_config(self) -> Optional[QueryJobConfig]: """Default ``QueryJobConfig`` or ``None``. @@ -3532,13 +3547,6 @@ def query_and_wait( ) -> RowIterator: """Run the query, wait for it to finish, and return the results. - While ``jobCreationMode=JOB_CREATION_OPTIONAL`` is in preview in the - ``jobs.query`` REST API, use the default ``jobCreationMode`` unless - the environment variable ``QUERY_PREVIEW_ENABLED=true``. After - ``jobCreationMode`` is GA, this method will always use - ``jobCreationMode=JOB_CREATION_OPTIONAL``. See: - https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs/query - Args: query (str): SQL query to be executed. Defaults to the standard SQL @@ -4134,7 +4142,7 @@ def _list_rows_from_query_results( rows that were affected. query (Optional[str]): The query text used. - total_bytes_processed (Optinal[int]): + total_bytes_processed (Optional[int]): total bytes processed from job statistics, if present. Returns: diff --git a/google/cloud/bigquery/enums.py b/google/cloud/bigquery/enums.py index 203ea3c7b..4cb7a056d 100644 --- a/google/cloud/bigquery/enums.py +++ b/google/cloud/bigquery/enums.py @@ -407,3 +407,22 @@ class BigLakeTableFormat(object): ICEBERG = "ICEBERG" """Apache Iceberg format.""" + + +class JobCreationMode(object): + """Documented values for Job Creation Mode.""" + + JOB_CREATION_MODE_UNSPECIFIED = "JOB_CREATION_MODE_UNSPECIFIED" + """Job creation mode is unspecified.""" + + JOB_CREATION_REQUIRED = "JOB_CREATION_REQUIRED" + """Job creation is always required.""" + + JOB_CREATION_OPTIONAL = "JOB_CREATION_OPTIONAL" + """Job creation is optional. + + Returning immediate results is prioritized. + BigQuery will automatically determine if a Job needs to be created. + The conditions under which BigQuery can decide to not create a Job are + subject to change. + """ diff --git a/google/cloud/bigquery/job/query.py b/google/cloud/bigquery/job/query.py index f9b99b7fb..954a46963 100644 --- a/google/cloud/bigquery/job/query.py +++ b/google/cloud/bigquery/job/query.py @@ -1529,7 +1529,7 @@ def result( # type: ignore # (incompatible with supertype) a DDL query, an ``_EmptyRowIterator`` instance is returned. Raises: - google.cloud.exceptions.GoogleAPICallError: + google.api_core.exceptions.GoogleAPICallError: If the job failed and retries aren't successful. concurrent.futures.TimeoutError: If the job did not complete in the given timeout. diff --git a/google/cloud/bigquery/table.py b/google/cloud/bigquery/table.py index 3f472c490..3b1334bd3 100644 --- a/google/cloud/bigquery/table.py +++ b/google/cloud/bigquery/table.py @@ -44,7 +44,7 @@ import geopandas # type: ignore except ImportError: geopandas = None -else: +finally: _COORDINATE_REFERENCE_SYSTEM = "EPSG:4326" try: @@ -1786,7 +1786,7 @@ class RowIterator(HTTPIterator): the first page is requested. query (Optional[str]): The query text used. - total_bytes_processed (Optinal[int]): + total_bytes_processed (Optional[int]): total bytes processed from job statistics, if present. """ diff --git a/google/cloud/bigquery/version.py b/google/cloud/bigquery/version.py index 8304ac025..9e1393854 100644 --- a/google/cloud/bigquery/version.py +++ b/google/cloud/bigquery/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "3.33.0" +__version__ = "3.34.0" diff --git a/noxfile.py b/noxfile.py index 575bbb100..6807b7ee4 100644 --- a/noxfile.py +++ b/noxfile.py @@ -163,8 +163,7 @@ def unit_noextras(session): # so that it continues to be an optional dependency. # https://github.com/googleapis/python-bigquery/issues/1877 if session.python == UNIT_TEST_PYTHON_VERSIONS[0]: - session.install("pyarrow==4.0.0") - + session.install("pyarrow==4.0.0", "numpy==1.20.2") default(session, install_extras=False) diff --git a/owlbot.py b/owlbot.py index 60759adbe..80cf9d6e3 100644 --- a/owlbot.py +++ b/owlbot.py @@ -65,6 +65,7 @@ templated_files, excludes=[ "noxfile.py", + "renovate.json", "docs/multiprocessing.rst", "docs/index.rst", ".coveragerc", @@ -76,7 +77,7 @@ ".kokoro/continuous/prerelease-deps.cfg", ".kokoro/samples/python3.7/**", ".kokoro/samples/python3.8/**", - ".github/workflows", # exclude gh actions as credentials are needed for tests + ".github/workflows/**", # exclude gh actions as credentials are needed for tests "README.rst", ], ) diff --git a/renovate.json b/renovate.json index c7875c469..3ea143d4c 100644 --- a/renovate.json +++ b/renovate.json @@ -5,8 +5,15 @@ ":preserveSemverRanges", ":disableDependencyDashboard" ], - "ignorePaths": [".pre-commit-config.yaml", ".kokoro/requirements.txt", "setup.py", ".github/workflows/unittest.yml"], + "ignorePaths": [".pre-commit-config.yaml", ".kokoro/requirements.txt", "setup.py", ".github/workflows/unittest.yml", ".github/workflows/docs.yml"], "pip_requirements": { "fileMatch": ["requirements-test.txt", "samples/[\\S/]*constraints.txt", "samples/[\\S/]*constraints-test.txt"] - } + }, + "packageRules": [ + { + "matchFileNames": ["pyproject.toml"], + "matchStrings": ["matplotlib (.*); python_version == '3.9'"], + "allowedVersions": ">= 3.7.1, <= 3.9.2" + } + ] } diff --git a/samples/client_query_shortmode.py b/samples/client_query_job_optional.py similarity index 69% rename from samples/client_query_shortmode.py rename to samples/client_query_job_optional.py index 50446dc48..6321aea35 100644 --- a/samples/client_query_shortmode.py +++ b/samples/client_query_job_optional.py @@ -13,16 +13,18 @@ # limitations under the License. -def client_query_shortmode() -> None: - # [START bigquery_query_shortquery] - # This example demonstrates issuing a query that may be run in short query mode. - # - # To enable the short query mode preview feature, the QUERY_PREVIEW_ENABLED - # environmental variable should be set to `TRUE`. +def client_query_job_optional() -> None: + # [START bigquery_query_job_optional] + # This example demonstrates executing a query without requiring an associated + # job. from google.cloud import bigquery + from google.cloud.bigquery.enums import JobCreationMode - # Construct a BigQuery client object. - client = bigquery.Client() + # Construct a BigQuery client object, specifying that the library should + # avoid creating jobs when possible. + client = bigquery.Client( + default_job_creation_mode=JobCreationMode.JOB_CREATION_OPTIONAL + ) query = """ SELECT @@ -44,10 +46,12 @@ def client_query_shortmode() -> None: if rows.job_id is not None: print("Query was run with job state. Job ID: {}".format(rows.job_id)) else: - print("Query was run in short mode. Query ID: {}".format(rows.query_id)) + print( + "Query was run without creating a job. Query ID: {}".format(rows.query_id) + ) print("The query data:") for row in rows: # Row values can be accessed by field name or index. print("name={}, gender={}, total={}".format(row[0], row[1], row["total"])) - # [END bigquery_query_shortquery] + # [END bigquery_query_job_optional] diff --git a/samples/client_query_w_array_params.py b/samples/client_query_w_array_params.py index 25592a94a..e9d759f61 100644 --- a/samples/client_query_w_array_params.py +++ b/samples/client_query_w_array_params.py @@ -35,8 +35,8 @@ def client_query_w_array_params() -> None: bigquery.ArrayQueryParameter("states", "STRING", ["WA", "WI", "WV", "WY"]), ] ) - query_job = client.query(query, job_config=job_config) # Make an API request. + rows = client.query_and_wait(query, job_config=job_config) # Make an API request. - for row in query_job: + for row in rows: print("{}: \t{}".format(row.name, row.count)) # [END bigquery_query_params_arrays] diff --git a/samples/desktopapp/requirements-test.txt b/samples/desktopapp/requirements-test.txt index 6abea3b4d..2ad35b418 100644 --- a/samples/desktopapp/requirements-test.txt +++ b/samples/desktopapp/requirements-test.txt @@ -1,4 +1,4 @@ -google-cloud-testutils==1.6.2 +google-cloud-testutils==1.6.4 pytest==8.3.5 mock==5.2.0 -pytest-xdist==3.6.1 +pytest-xdist==3.7.0 diff --git a/samples/desktopapp/requirements.txt b/samples/desktopapp/requirements.txt index 743d0fe35..4a5b75346 100644 --- a/samples/desktopapp/requirements.txt +++ b/samples/desktopapp/requirements.txt @@ -1,2 +1,2 @@ -google-cloud-bigquery==3.32.0 +google-cloud-bigquery==3.33.0 google-auth-oauthlib==1.2.2 diff --git a/samples/geography/requirements-test.txt b/samples/geography/requirements-test.txt index 7b01ce8ac..3ca365401 100644 --- a/samples/geography/requirements-test.txt +++ b/samples/geography/requirements-test.txt @@ -1,3 +1,3 @@ pytest==8.3.5 mock==5.2.0 -pytest-xdist==3.6.1 +pytest-xdist==3.7.0 diff --git a/samples/geography/requirements.txt b/samples/geography/requirements.txt index 434a594cb..7a0946fae 100644 --- a/samples/geography/requirements.txt +++ b/samples/geography/requirements.txt @@ -3,7 +3,7 @@ certifi==2025.4.26 cffi==1.17.1 charset-normalizer==3.4.2 click===8.1.8; python_version == '3.9' -click==8.2.0; python_version >= '3.10' +click==8.2.1; python_version >= '3.10' click-plugins==1.1.1 cligj==0.7.2 db-dtypes==1.4.3 @@ -11,8 +11,8 @@ Fiona==1.10.1 geojson==3.2.0 geopandas==1.0.1 google-api-core==2.24.2 -google-auth==2.40.1 -google-cloud-bigquery==3.32.0 +google-auth==2.40.2 +google-cloud-bigquery==3.33.0 google-cloud-bigquery-storage==2.31.0 google-cloud-core==2.4.3 google-crc32c==1.7.1 @@ -36,7 +36,7 @@ PyYAML==6.0.2 requests==2.32.3 rsa==4.9.1 Shapely===2.0.7; python_version == '3.9' -Shapely==2.1.0; python_version >= '3.10' +Shapely==2.1.1; python_version >= '3.10' six==1.17.0 typing-extensions==4.13.2 typing-inspect==0.9.0 diff --git a/samples/magics/requirements-test.txt b/samples/magics/requirements-test.txt index 6abea3b4d..2ad35b418 100644 --- a/samples/magics/requirements-test.txt +++ b/samples/magics/requirements-test.txt @@ -1,4 +1,4 @@ -google-cloud-testutils==1.6.2 +google-cloud-testutils==1.6.4 pytest==8.3.5 mock==5.2.0 -pytest-xdist==3.6.1 +pytest-xdist==3.7.0 diff --git a/samples/magics/requirements.txt b/samples/magics/requirements.txt index bb60f2a67..7d0c91e3d 100644 --- a/samples/magics/requirements.txt +++ b/samples/magics/requirements.txt @@ -1,6 +1,6 @@ -bigquery_magics==0.9.0 +bigquery_magics==0.10.0 db-dtypes==1.4.3 -google.cloud.bigquery==3.32.0 +google.cloud.bigquery==3.33.0 google-cloud-bigquery-storage==2.31.0 ipython===8.18.1 pandas==2.2.3 diff --git a/samples/notebooks/requirements-test.txt b/samples/notebooks/requirements-test.txt index 6abea3b4d..2ad35b418 100644 --- a/samples/notebooks/requirements-test.txt +++ b/samples/notebooks/requirements-test.txt @@ -1,4 +1,4 @@ -google-cloud-testutils==1.6.2 +google-cloud-testutils==1.6.4 pytest==8.3.5 mock==5.2.0 -pytest-xdist==3.6.1 +pytest-xdist==3.7.0 diff --git a/samples/notebooks/requirements.txt b/samples/notebooks/requirements.txt index 17f43bf78..9f131e5b8 100644 --- a/samples/notebooks/requirements.txt +++ b/samples/notebooks/requirements.txt @@ -1,6 +1,6 @@ -bigquery-magics==0.9.0 +bigquery-magics==0.10.0 db-dtypes==1.4.3 -google-cloud-bigquery==3.32.0 +google-cloud-bigquery==3.33.0 google-cloud-bigquery-storage==2.31.0 ipython===8.18.1; python_version == '3.9' ipython==9.2.0; python_version >= '3.10' diff --git a/samples/snippets/requirements-test.txt b/samples/snippets/requirements-test.txt index 6760e1228..767f71fb1 100644 --- a/samples/snippets/requirements-test.txt +++ b/samples/snippets/requirements-test.txt @@ -1,5 +1,5 @@ # samples/snippets should be runnable with no "extras" -google-cloud-testutils==1.6.2 +google-cloud-testutils==1.6.4 pytest==8.3.5 mock==5.2.0 -pytest-xdist==3.6.1 +pytest-xdist==3.7.0 diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index c31815d69..dae43eff3 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,2 +1,2 @@ # samples/snippets should be runnable with no "extras" -google-cloud-bigquery==3.32.0 +google-cloud-bigquery==3.33.0 diff --git a/samples/tests/test_client_query_shortmode.py b/samples/tests/test_client_query_job_optional.py similarity index 85% rename from samples/tests/test_client_query_shortmode.py rename to samples/tests/test_client_query_job_optional.py index 41132f24c..0e0b2cf19 100644 --- a/samples/tests/test_client_query_shortmode.py +++ b/samples/tests/test_client_query_job_optional.py @@ -1,4 +1,4 @@ -# Copyright 2024 Google LLC +# Copyright 2025 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,13 +14,13 @@ import typing -from .. import client_query_shortmode +from .. import client_query_job_optional if typing.TYPE_CHECKING: import pytest def test_client_query_shortmode(capsys: "pytest.CaptureFixture[str]") -> None: - client_query_shortmode.client_query_shortmode() + client_query_job_optional.client_query_job_optional() out, err = capsys.readouterr() assert "Query was run" in out diff --git a/testing/constraints-3.9.txt b/testing/constraints-3.9.txt index 60a155f0d..f61c0cf09 100644 --- a/testing/constraints-3.9.txt +++ b/testing/constraints-3.9.txt @@ -20,6 +20,7 @@ ipykernel==6.2.0 opentelemetry-api==1.1.0 opentelemetry-instrumentation==0.20b0 opentelemetry-sdk==1.1.0 +numpy==1.20.2 packaging==24.2.0 pandas==1.3.0 pandas-gbq==0.26.1 diff --git a/tests/unit/test__job_helpers.py b/tests/unit/test__job_helpers.py index 4fa093c69..417f911b8 100644 --- a/tests/unit/test__job_helpers.py +++ b/tests/unit/test__job_helpers.py @@ -554,13 +554,9 @@ def test_query_and_wait_retries_job_times_out(): ) -def test_query_and_wait_sets_job_creation_mode(monkeypatch: pytest.MonkeyPatch): - monkeypatch.setenv( - "QUERY_PREVIEW_ENABLED", - # The comparison should be case insensitive. - "TrUe", - ) +def test_query_and_wait_sets_job_creation_mode(): client = mock.create_autospec(Client) + client.default_job_creation_mode = "JOB_CREATION_OPTIONAL" client._call_api.return_value = { "jobReference": { "projectId": "response-project", @@ -642,6 +638,7 @@ def test_query_and_wait_sets_location(): "useInt64Timestamp": True, }, "requestId": mock.ANY, + "jobCreationMode": mock.ANY, }, timeout=None, ) @@ -658,6 +655,7 @@ def test_query_and_wait_sets_location(): ) def test_query_and_wait_sets_max_results(max_results, page_size, expected): client = mock.create_autospec(Client) + client.default_job_creation_mode = None client._call_api.return_value = { "jobReference": { "projectId": "response-project", @@ -703,6 +701,7 @@ def test_query_and_wait_sets_max_results(max_results, page_size, expected): def test_query_and_wait_caches_completed_query_results_one_page(): client = mock.create_autospec(Client) + client.default_job_creation_mode = None client._call_api.return_value = { "jobReference": { "projectId": "response-project", @@ -768,6 +767,7 @@ def test_query_and_wait_caches_completed_query_results_one_page(): def test_query_and_wait_caches_completed_query_results_one_page_no_rows(): client = mock.create_autospec(Client) + client.default_job_creation_mode = None client._call_api.return_value = { "jobReference": { "projectId": "response-project", diff --git a/tests/unit/test__pandas_helpers.py b/tests/unit/test__pandas_helpers.py index d87c65581..bc94f5f54 100644 --- a/tests/unit/test__pandas_helpers.py +++ b/tests/unit/test__pandas_helpers.py @@ -1856,6 +1856,7 @@ def test__download_table_bqstorage_shuts_down_workers( Make sure that when the top-level iterator goes out of scope (is deleted), the child threads are also stopped. """ + pytest.importorskip("google.cloud.bigquery_storage_v1") from google.cloud.bigquery import dataset from google.cloud.bigquery import table import google.cloud.bigquery_storage_v1.reader diff --git a/tests/unit/test__pyarrow_helpers.py b/tests/unit/test__pyarrow_helpers.py index 06fc2eb85..c12a526de 100644 --- a/tests/unit/test__pyarrow_helpers.py +++ b/tests/unit/test__pyarrow_helpers.py @@ -14,7 +14,7 @@ import pytest - +numpy = pytest.importorskip("numpy") pyarrow = pytest.importorskip("pyarrow", minversion="3.0.0") diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index 468068321..8ce8d2cbd 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -213,6 +213,17 @@ def test_ctor_w_client_options_universe(self): ) self.assertEqual(client._connection.API_BASE_URL, "https://bigquery.foo.com") + def test_ctor_w_job_creation_mode(self): + creds = _make_credentials() + http = object() + client = self._make_one( + project=self.PROJECT, + credentials=creds, + _http=http, + default_job_creation_mode="foo", + ) + self.assertEqual(client.default_job_creation_mode, "foo") + def test_ctor_w_location(self): from google.cloud.bigquery._http import Connection diff --git a/tests/unit/test_dbapi__helpers.py b/tests/unit/test_dbapi__helpers.py index 7e1da0034..9907df97b 100644 --- a/tests/unit/test_dbapi__helpers.py +++ b/tests/unit/test_dbapi__helpers.py @@ -210,6 +210,7 @@ def test_empty_iterable(self): self.assertEqual(list(result), []) def test_non_empty_iterable(self): + pytest.importorskip("numpy") pytest.importorskip("pyarrow") from tests.unit.helpers import _to_pyarrow diff --git a/tests/unit/test_magics.py b/tests/unit/test_magics.py index a9a12283b..814150693 100644 --- a/tests/unit/test_magics.py +++ b/tests/unit/test_magics.py @@ -1276,6 +1276,11 @@ def test_bigquery_magic_with_no_query_cache(monkeypatch): bigquery.load_ipython_extension(ip) conn = make_connection() monkeypatch.setattr(magics.context, "_connection", conn) + monkeypatch.setattr( + magics.context, + "credentials", + mock.create_autospec(google.auth.credentials.Credentials, instance=True), + ) monkeypatch.setattr(magics.context, "project", "project-from-context") # --no_query_cache option should override context. diff --git a/tests/unit/test_table.py b/tests/unit/test_table.py index 4791c6511..eb2c8d9ec 100644 --- a/tests/unit/test_table.py +++ b/tests/unit/test_table.py @@ -2416,6 +2416,7 @@ def test_to_arrow_error_if_pyarrow_is_none(self): row_iterator.to_arrow() def test_to_arrow(self): + pytest.importorskip("numpy") pyarrow = pytest.importorskip("pyarrow") row_iterator = self._make_one() tbl = row_iterator.to_arrow() @@ -2423,6 +2424,7 @@ def test_to_arrow(self): self.assertEqual(tbl.num_rows, 0) def test_to_arrow_iterable(self): + pytest.importorskip("numpy") pyarrow = pytest.importorskip( "pyarrow", minversion=self.PYARROW_MINIMUM_VERSION ) @@ -3089,6 +3091,7 @@ def test_to_arrow_iterable_w_bqstorage(self): bqstorage_client._transport.grpc_channel.close.assert_not_called() def test_to_arrow(self): + pytest.importorskip("numpy") pyarrow = pytest.importorskip( "pyarrow", minversion=self.PYARROW_MINIMUM_VERSION ) @@ -3173,6 +3176,7 @@ def test_to_arrow(self): ) def test_to_arrow_w_nulls(self): + pytest.importorskip("numpy") pyarrow = pytest.importorskip( "pyarrow", minversion=self.PYARROW_MINIMUM_VERSION ) @@ -3209,6 +3213,7 @@ def test_to_arrow_w_nulls(self): self.assertEqual(ages, [32, 29, None, 111]) def test_to_arrow_w_unknown_type(self): + pytest.importorskip("numpy") pyarrow = pytest.importorskip( "pyarrow", minversion=self.PYARROW_MINIMUM_VERSION ) @@ -3254,6 +3259,7 @@ def test_to_arrow_w_unknown_type(self): self.assertTrue(all("sport" in str(warning) for warning in warned)) def test_to_arrow_w_empty_table(self): + pytest.importorskip("numpy") pyarrow = pytest.importorskip( "pyarrow", minversion=self.PYARROW_MINIMUM_VERSION ) @@ -3295,6 +3301,7 @@ def test_to_arrow_w_empty_table(self): self.assertEqual(child_field.type.value_type[1].name, "age") def test_to_arrow_max_results_w_explicit_bqstorage_client_warning(self): + pytest.importorskip("numpy") pytest.importorskip("pyarrow") pytest.importorskip("google.cloud.bigquery_storage") from google.cloud.bigquery.schema import SchemaField @@ -3337,6 +3344,7 @@ def test_to_arrow_max_results_w_explicit_bqstorage_client_warning(self): mock_client._ensure_bqstorage_client.assert_not_called() def test_to_arrow_max_results_w_create_bqstorage_client_no_warning(self): + pytest.importorskip("numpy") pytest.importorskip("pyarrow") pytest.importorskip("google.cloud.bigquery_storage") from google.cloud.bigquery.schema import SchemaField @@ -3375,6 +3383,7 @@ def test_to_arrow_max_results_w_create_bqstorage_client_no_warning(self): mock_client._ensure_bqstorage_client.assert_not_called() def test_to_arrow_w_bqstorage(self): + pytest.importorskip("numpy") pyarrow = pytest.importorskip("pyarrow") pytest.importorskip("google.cloud.bigquery_storage") from google.cloud.bigquery import schema @@ -3458,6 +3467,7 @@ def test_to_arrow_w_bqstorage(self): bqstorage_client._transport.grpc_channel.close.assert_not_called() def test_to_arrow_w_bqstorage_creates_client(self): + pytest.importorskip("numpy") pytest.importorskip("pyarrow") pytest.importorskip("google.cloud.bigquery_storage") from google.cloud.bigquery import schema @@ -3491,6 +3501,7 @@ def test_to_arrow_w_bqstorage_creates_client(self): bqstorage_client._transport.grpc_channel.close.assert_called_once() def test_to_arrow_ensure_bqstorage_client_wo_bqstorage(self): + pytest.importorskip("numpy") pyarrow = pytest.importorskip( "pyarrow", minversion=self.PYARROW_MINIMUM_VERSION ) @@ -3524,6 +3535,7 @@ def mock_verify_version(raise_if_error: bool = False): self.assertEqual(tbl.num_rows, 2) def test_to_arrow_w_bqstorage_no_streams(self): + pytest.importorskip("numpy") pyarrow = pytest.importorskip("pyarrow") pytest.importorskip("google.cloud.bigquery_storage") from google.cloud.bigquery import schema @@ -3563,6 +3575,7 @@ def test_to_arrow_w_bqstorage_no_streams(self): self.assertEqual(actual_table.schema[2].name, "colB") def test_to_arrow_progress_bar(self): + pytest.importorskip("numpy") pytest.importorskip("pyarrow") pytest.importorskip("tqdm") pytest.importorskip("tqdm.notebook") @@ -3696,6 +3709,7 @@ def test_to_dataframe_iterable_with_dtypes(self): self.assertEqual(df_2["age"][0], 33) def test_to_dataframe_iterable_w_bqstorage(self): + pytest.importorskip("numpy") pandas = pytest.importorskip("pandas") pyarrow = pytest.importorskip("pyarrow") pytest.importorskip("google.cloud.bigquery_storage") @@ -3770,6 +3784,7 @@ def test_to_dataframe_iterable_w_bqstorage(self): bqstorage_client._transport.grpc_channel.close.assert_not_called() def test_to_dataframe_iterable_w_bqstorage_max_results_warning(self): + pytest.importorskip("numpy") pandas = pytest.importorskip("pandas") pytest.importorskip("google.cloud.bigquery_storage") from google.cloud.bigquery import schema @@ -4513,7 +4528,7 @@ def test_to_dataframe_w_none_dtypes_mapper(self): def test_to_dataframe_w_unsupported_dtypes_mapper(self): pytest.importorskip("pandas") - import numpy + numpy = pytest.importorskip("numpy") from google.cloud.bigquery.schema import SchemaField schema = [ @@ -4797,6 +4812,7 @@ def test_to_dataframe_max_results_w_create_bqstorage_client_no_warning(self): mock_client._ensure_bqstorage_client.assert_not_called() def test_to_dataframe_w_bqstorage_creates_client(self): + pytest.importorskip("numpy") pytest.importorskip("pandas") pytest.importorskip("google.cloud.bigquery_storage") from google.cloud.bigquery import schema @@ -4830,6 +4846,7 @@ def test_to_dataframe_w_bqstorage_creates_client(self): bqstorage_client._transport.grpc_channel.close.assert_called_once() def test_to_dataframe_w_bqstorage_no_streams(self): + pytest.importorskip("numpy") pytest.importorskip("pandas") pytest.importorskip("google.cloud.bigquery_storage") from google.cloud.bigquery import schema @@ -4858,6 +4875,7 @@ def test_to_dataframe_w_bqstorage_no_streams(self): self.assertTrue(got.empty) def test_to_dataframe_w_bqstorage_logs_session(self): + pytest.importorskip("numpy") pytest.importorskip("google.cloud.bigquery_storage") pytest.importorskip("pandas") pytest.importorskip("pyarrow") @@ -4882,6 +4900,7 @@ def test_to_dataframe_w_bqstorage_logs_session(self): ) def test_to_dataframe_w_bqstorage_empty_streams(self): + pytest.importorskip("numpy") pytest.importorskip("google.cloud.bigquery_storage") pytest.importorskip("pandas") pyarrow = pytest.importorskip("pyarrow") @@ -4936,6 +4955,7 @@ def test_to_dataframe_w_bqstorage_empty_streams(self): self.assertTrue(got.empty) def test_to_dataframe_w_bqstorage_nonempty(self): + pytest.importorskip("numpy") pytest.importorskip("google.cloud.bigquery_storage") pytest.importorskip("pandas") pyarrow = pytest.importorskip("pyarrow") @@ -5018,6 +5038,7 @@ def test_to_dataframe_w_bqstorage_nonempty(self): bqstorage_client._transport.grpc_channel.close.assert_not_called() def test_to_dataframe_w_bqstorage_multiple_streams_return_unique_index(self): + pytest.importorskip("numpy") bigquery_storage = pytest.importorskip("google.cloud.bigquery_storage") pytest.importorskip("pandas") pyarrow = pytest.importorskip("pyarrow") @@ -5070,6 +5091,7 @@ def test_to_dataframe_w_bqstorage_multiple_streams_return_unique_index(self): self.assertTrue(got.index.is_unique) def test_to_dataframe_w_bqstorage_updates_progress_bar(self): + pytest.importorskip("numpy") bigquery_storage = pytest.importorskip("google.cloud.bigquery_storage") pytest.importorskip("pandas") pyarrow = pytest.importorskip("pyarrow") @@ -5147,6 +5169,7 @@ def blocking_to_arrow(*args, **kwargs): tqdm_mock().close.assert_called_once() def test_to_dataframe_w_bqstorage_exits_on_keyboardinterrupt(self): + pytest.importorskip("numpy") bigquery_storage = pytest.importorskip("google.cloud.bigquery_storage") pytest.importorskip("pandas") pyarrow = pytest.importorskip("pyarrow") @@ -5322,6 +5345,7 @@ def test_to_dataframe_w_bqstorage_snapshot(self): row_iterator.to_dataframe(bqstorage_client) def test_to_dataframe_concat_categorical_dtype_w_pyarrow(self): + pytest.importorskip("numpy") pytest.importorskip("google.cloud.bigquery_storage") pandas = pytest.importorskip("pandas") pyarrow = pytest.importorskip("pyarrow") @@ -5604,7 +5628,7 @@ def test_rowiterator_to_geodataframe_delegation(self, to_dataframe): """ pandas = pytest.importorskip("pandas") geopandas = pytest.importorskip("geopandas") - import numpy + numpy = pytest.importorskip("numpy") from shapely import wkt row_iterator = self._make_one_from_data( diff --git a/tests/unit/test_table_arrow.py b/tests/unit/test_table_arrow.py index 830c4ceb7..fdd1b7b78 100644 --- a/tests/unit/test_table_arrow.py +++ b/tests/unit/test_table_arrow.py @@ -18,7 +18,8 @@ import google.cloud.bigquery.table -pyarrow = pytest.importorskip("pyarrow", minversion="3.0.0") +pytest.importorskip("numpy") +pytest.importorskip("pyarrow", minversion="3.0.0") def test_to_arrow_with_jobs_query_response():