diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index ff5126c18..8cb43804d 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -1,3 +1,16 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:dfa9b663b32de8b5b327e32c1da665a80de48876558dd58091d8160c60ad7355 + digest: sha256:ed1f9983d5a935a89fe8085e8bb97d94e41015252c5b6c9771257cf8624367e6 diff --git a/.github/sync-repo-settings.yaml b/.github/sync-repo-settings.yaml index 01affbae5..73cc3bcef 100644 --- a/.github/sync-repo-settings.yaml +++ b/.github/sync-repo-settings.yaml @@ -14,7 +14,6 @@ branchProtectionRules: - 'Kokoro snippets-3.8' - 'cla/google' - 'Samples - Lint' - - 'Samples - Python 3.6' - 'Samples - Python 3.7' - 'Samples - Python 3.8' - pattern: v3 @@ -26,6 +25,5 @@ branchProtectionRules: - 'Kokoro snippets-3.8' - 'cla/google' - 'Samples - Lint' - - 'Samples - Python 3.6' - 'Samples - Python 3.7' - 'Samples - Python 3.8' diff --git a/.kokoro/release.sh b/.kokoro/release.sh index 3abba6e06..b030caeef 100755 --- a/.kokoro/release.sh +++ b/.kokoro/release.sh @@ -26,7 +26,7 @@ python3 -m pip install --upgrade twine wheel setuptools export PYTHONUNBUFFERED=1 # Move into the package, build the distribution and upload. -TWINE_PASSWORD=$(cat "${KOKORO_GFILE_DIR}/secret_manager/google-cloud-pypi-token") +TWINE_PASSWORD=$(cat "${KOKORO_KEYSTORE_DIR}/73713_google-cloud-pypi-token-keystore-1") cd github/python-bigquery python3 setup.py sdist bdist_wheel twine upload --username __token__ --password "${TWINE_PASSWORD}" dist/* diff --git a/.kokoro/release/common.cfg b/.kokoro/release/common.cfg index 922d7fe50..6ae81b743 100644 --- a/.kokoro/release/common.cfg +++ b/.kokoro/release/common.cfg @@ -23,8 +23,18 @@ env_vars: { value: "github/python-bigquery/.kokoro/release.sh" } +# Fetch PyPI password +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73713 + keyname: "google-cloud-pypi-token-keystore-1" + } + } +} + # Tokens needed to report release status back to GitHub env_vars: { key: "SECRET_MANAGER_KEYS" - value: "releasetool-publish-reporter-app,releasetool-publish-reporter-googleapis-installation,releasetool-publish-reporter-pem,google-cloud-pypi-token" + value: "releasetool-publish-reporter-app,releasetool-publish-reporter-googleapis-installation,releasetool-publish-reporter-pem" } diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e69fa621..e83cb9788 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,24 @@ [1]: https://pypi.org/project/google-cloud-bigquery/#history +## [2.33.0](https://github.com/googleapis/python-bigquery/compare/v2.32.0...v2.33.0) (2022-02-16) + + +### Features + +* add `--no_query_cache` option to `%%bigquery` magics to disable query cache ([#1141](https://github.com/googleapis/python-bigquery/issues/1141)) ([7dd30af](https://github.com/googleapis/python-bigquery/commit/7dd30af41b8a595b96176c964ba14aa41645ef0d)) + + +### Bug Fixes + +* return 403 when VPC-SC violation happens ([#1131](https://github.com/googleapis/python-bigquery/issues/1131)) ([f5daa9b](https://github.com/googleapis/python-bigquery/commit/f5daa9b41377a58cb3220bb2ab7c72adc6462196)) + + +### Documentation + +* reference BigQuery REST API defaults in `LoadJobConfig` descrip… ([#1132](https://github.com/googleapis/python-bigquery/issues/1132)) ([18d9580](https://github.com/googleapis/python-bigquery/commit/18d958062721d6be81e7bd7a5bd66f277344a864)) +* show common job properties in `get_job` and `cancel_job` samples ([#1137](https://github.com/googleapis/python-bigquery/issues/1137)) ([8edc10d](https://github.com/googleapis/python-bigquery/commit/8edc10d019bd96defebc4f92a47774901e9b956f)) + ## [2.32.0](https://github.com/googleapis/python-bigquery/compare/v2.31.0...v2.32.0) (2022-01-12) diff --git a/docs/snippets.py b/docs/snippets.py index c62001fc0..f67823249 100644 --- a/docs/snippets.py +++ b/docs/snippets.py @@ -756,45 +756,6 @@ def test_client_query_total_rows(client, capsys): assert "Got 100 rows." in out -def test_manage_job(client): - sql = """ - SELECT corpus - FROM `bigquery-public-data.samples.shakespeare` - GROUP BY corpus; - """ - location = "us" - job = client.query(sql, location=location) - job_id = job.job_id - - # [START bigquery_cancel_job] - # TODO(developer): Uncomment the lines below and replace with your values. - # from google.cloud import bigquery - # client = bigquery.Client() - # job_id = 'bq-job-123x456-123y123z123c' # replace with your job ID - # location = 'us' # replace with your location - - job = client.cancel_job(job_id, location=location) - # [END bigquery_cancel_job] - - # [START bigquery_get_job] - # TODO(developer): Uncomment the lines below and replace with your values. - # from google.cloud import bigquery - # client = bigquery.Client() - # job_id = 'bq-job-123x456-123y123z123c' # replace with your job ID - # location = 'us' # replace with your location - - job = client.get_job(job_id, location=location) # API request - - # Print selected job properties - print("Details for job {} running in {}:".format(job_id, location)) - print( - "\tType: {}\n\tState: {}\n\tCreated: {}".format( - job.job_type, job.state, job.created - ) - ) - # [END bigquery_get_job] - - def test_query_external_gcs_permanent_table(client, to_delete): dataset_id = "query_external_gcs_{}".format(_millis()) project = client.project diff --git a/google/cloud/bigquery/job/base.py b/google/cloud/bigquery/job/base.py index 97acab5d2..86701e295 100644 --- a/google/cloud/bigquery/job/base.py +++ b/google/cloud/bigquery/job/base.py @@ -45,6 +45,7 @@ "invalidQuery": http.client.BAD_REQUEST, "notFound": http.client.NOT_FOUND, "notImplemented": http.client.NOT_IMPLEMENTED, + "policyViolation": http.client.FORBIDDEN, "quotaExceeded": http.client.FORBIDDEN, "rateLimitExceeded": http.client.FORBIDDEN, "resourceInUse": http.client.BAD_REQUEST, diff --git a/google/cloud/bigquery/job/load.py b/google/cloud/bigquery/job/load.py index b12c3e621..2d68f7f71 100644 --- a/google/cloud/bigquery/job/load.py +++ b/google/cloud/bigquery/job/load.py @@ -33,9 +33,21 @@ class LoadJobConfig(_JobConfig): """Configuration options for load jobs. - All properties in this class are optional. Values which are :data:`None` -> - server defaults. Set properties on the constructed configuration by using - the property name as the name of a keyword argument. + Set properties on the constructed configuration by using the property name + as the name of a keyword argument. Values which are unset or :data:`None` + use the BigQuery REST API default values. See the `BigQuery REST API + reference documentation + `_ + for a list of default values. + + Required options differ based on the + :attr:`~google.cloud.bigquery.job.LoadJobConfig.source_format` value. + For example, the BigQuery API's default value for + :attr:`~google.cloud.bigquery.job.LoadJobConfig.source_format` is ``"CSV"``. + When loading a CSV file, either + :attr:`~google.cloud.bigquery.job.LoadJobConfig.schema` must be set or + :attr:`~google.cloud.bigquery.job.LoadJobConfig.autodetect` must be set to + :data:`True`. """ def __init__(self, **kwargs): diff --git a/google/cloud/bigquery/magics/magics.py b/google/cloud/bigquery/magics/magics.py index 5af0a3b51..7b4d584fb 100644 --- a/google/cloud/bigquery/magics/magics.py +++ b/google/cloud/bigquery/magics/magics.py @@ -35,6 +35,8 @@ A dataset and table to store the query results. If table does not exists, it will be created. If table already exists, its data will be overwritten. Variable should be in a format .. + * ``--no_query_cache`` (Optional[line argument]): + Do not use cached query results. * ``--project `` (Optional[line argument]): Project to use for running the query. Defaults to the context :attr:`~google.cloud.bigquery.magics.Context.project`. @@ -442,6 +444,12 @@ def _create_dataset_if_necessary(client, dataset_id): "this option's value in the context bqstorage_client_options." ), ) +@magic_arguments.argument( + "--no_query_cache", + action="store_true", + default=False, + help=("Do not use cached query results."), +) @magic_arguments.argument( "--use_bqstorage_api", action="store_true", @@ -642,6 +650,10 @@ def _cell_magic(line, query): job_config.use_legacy_sql = args.use_legacy_sql job_config.dry_run = args.dry_run + # Don't override context job config unless --no_query_cache is explicitly set. + if args.no_query_cache: + job_config.use_query_cache = False + if args.destination_table: split = args.destination_table.split(".") if len(split) != 2: diff --git a/google/cloud/bigquery/version.py b/google/cloud/bigquery/version.py index b8c5af9a2..17811b65f 100644 --- a/google/cloud/bigquery/version.py +++ b/google/cloud/bigquery/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.32.0" +__version__ = "2.33.0" diff --git a/google/cloud/bigquery_v2/types/model.py b/google/cloud/bigquery_v2/types/model.py index 440bc0805..84188e40c 100644 --- a/google/cloud/bigquery_v2/types/model.py +++ b/google/cloud/bigquery_v2/types/model.py @@ -341,8 +341,8 @@ class AggregateClassificationMetrics(proto.Message): threshold. f1_score (google.protobuf.wrappers_pb2.DoubleValue): The F1 score is an average of recall and - precision. For multiclass this is a macro- - averaged metric. + precision. For multiclass this is a + macro-averaged metric. log_loss (google.protobuf.wrappers_pb2.DoubleValue): Logarithmic Loss. For multiclass this is a macro-averaged metric. diff --git a/owlbot.py b/owlbot.py index e6f36905b..095759d48 100644 --- a/owlbot.py +++ b/owlbot.py @@ -116,6 +116,7 @@ # Include custom SNIPPETS_TESTS job for performance. # https://github.com/googleapis/python-bigquery/issues/191 ".kokoro/presubmit/presubmit.cfg", + ".github/workflows", # exclude gh actions as credentials are needed for tests ], ) diff --git a/samples/geography/noxfile.py b/samples/geography/noxfile.py index 3bbef5d54..20cdfc620 100644 --- a/samples/geography/noxfile.py +++ b/samples/geography/noxfile.py @@ -187,6 +187,7 @@ def _session_tests( ) -> None: # check for presence of tests test_list = glob.glob("*_test.py") + glob.glob("test_*.py") + test_list.extend(glob.glob("tests")) if len(test_list) == 0: print("No tests found, skipping directory.") else: diff --git a/samples/geography/requirements.txt b/samples/geography/requirements.txt index ca7e38f84..b07ba50bf 100644 --- a/samples/geography/requirements.txt +++ b/samples/geography/requirements.txt @@ -5,12 +5,12 @@ charset-normalizer==2.0.10 click==8.0.3 click-plugins==1.1.1 cligj==0.7.2 -dataclasses==0.6; python_version < '3.7' +dataclasses==0.8; python_version < '3.7' Fiona==1.8.20 geojson==2.5.0 geopandas==0.9.0; python_version < '3.7' geopandas==0.10.2; python_version >= '3.7' -google-api-core==2.3.2 +google-api-core==2.4.0 google-auth==2.3.3 google-cloud-bigquery==2.31.0 google-cloud-bigquery-storage==2.10.1 @@ -20,21 +20,19 @@ google-resumable-media==2.1.0 googleapis-common-protos==1.54.0 grpcio==1.43.0 idna==3.3 -libcst==0.3.23 +libcst==0.4.0 munch==2.5.0 mypy-extensions==0.4.3 packaging==21.3 pandas==1.1.5; python_version < '3.7' -pandas==1.3.4; python_version >= '3.7' +pandas==1.3.5; python_version >= '3.7' proto-plus==1.19.8 -protobuf==3.19.1 +protobuf==3.19.3 pyarrow==6.0.1 pyasn1==0.4.8 pyasn1-modules==0.2.8 pycparser==2.21 pyparsing==3.0.6 -pyproj==3.0.1; python_version < "3.7" -pyproj==3.1.0; python_version > "3.6" python-dateutil==2.8.2 pytz==2021.3 PyYAML==6.0 @@ -44,4 +42,4 @@ Shapely==1.8.0 six==1.16.0 typing-extensions==4.0.1 typing-inspect==0.7.1 -urllib3==1.26.7 +urllib3==1.26.8 diff --git a/samples/magics/noxfile.py b/samples/magics/noxfile.py index 3bbef5d54..20cdfc620 100644 --- a/samples/magics/noxfile.py +++ b/samples/magics/noxfile.py @@ -187,6 +187,7 @@ def _session_tests( ) -> None: # check for presence of tests test_list = glob.glob("*_test.py") + glob.glob("test_*.py") + test_list.extend(glob.glob("tests")) if len(test_list) == 0: print("No tests found, skipping directory.") else: diff --git a/samples/magics/requirements.txt b/samples/magics/requirements.txt index f4337e8fb..b47dcbc4f 100644 --- a/samples/magics/requirements.txt +++ b/samples/magics/requirements.txt @@ -2,10 +2,11 @@ google-cloud-bigquery-storage==2.10.1 google-auth-oauthlib==0.4.6 grpcio==1.43.0 ipython==7.16.1; python_version < '3.7' -ipython==7.29.0; python_version >= '3.7' +ipython==7.29.0; python_version == '3.7' +ipython==8.0.0; python_version >= '3.8' matplotlib==3.3.4; python_version < '3.7' -matplotlib==3.5.0rc1; python_version >= '3.7' +matplotlib==3.5.1; python_version >= '3.7' pandas==1.1.5; python_version < '3.7' -pandas==1.3.4; python_version >= '3.7' +pandas==1.3.5; python_version >= '3.7' pyarrow==6.0.1 pytz==2021.3 diff --git a/samples/snippets/manage_job_cancel.py b/samples/snippets/manage_job_cancel.py new file mode 100644 index 000000000..3e0fc5218 --- /dev/null +++ b/samples/snippets/manage_job_cancel.py @@ -0,0 +1,26 @@ +# Copyright 2016-2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START bigquery_cancel_job] +from google.cloud import bigquery + + +def cancel_job( + client: bigquery.Client, location: str = "us", job_id: str = "abcd-efgh-ijkl-mnop", +): + job = client.cancel_job(job_id, location=location) + print(f"{job.location}:{job.job_id} cancelled") + + +# [END bigquery_cancel_job] diff --git a/samples/snippets/manage_job_get.py b/samples/snippets/manage_job_get.py new file mode 100644 index 000000000..256d79e5b --- /dev/null +++ b/samples/snippets/manage_job_get.py @@ -0,0 +1,33 @@ +# Copyright 2016-2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START bigquery_get_job] +from google.cloud import bigquery + + +def get_job( + client: bigquery.Client, location: str = "us", job_id: str = "abcd-efgh-ijkl-mnop", +): + job = client.get_job(job_id, location=location) + + # All job classes have "location" and "job_id" string properties. + # Use these properties for job operations such as "cancel_job" and + # "delete_job". + print(f"{job.location}:{job.job_id}") + print(f"Type: {job.job_type}") + print(f"State: {job.state}") + print(f"Created: {job.created.isoformat()}") + + +# [END bigquery_get_job] diff --git a/samples/snippets/manage_job_test.py b/samples/snippets/manage_job_test.py new file mode 100644 index 000000000..745b7bbbe --- /dev/null +++ b/samples/snippets/manage_job_test.py @@ -0,0 +1,39 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from google.cloud import bigquery +import pytest + +import manage_job_cancel +import manage_job_get + + +def test_manage_job(capsys: pytest.CaptureFixture): + client = bigquery.Client() + sql = """ + SELECT corpus + FROM `bigquery-public-data.samples.shakespeare` + GROUP BY corpus; + """ + location = "us" + job = client.query(sql, location=location) + + manage_job_cancel.cancel_job(client, location=location, job_id=job.job_id) + out, _ = capsys.readouterr() + assert f"{job.location}:{job.job_id} cancelled" in out + + manage_job_get.get_job(client, location=location, job_id=job.job_id) + out, _ = capsys.readouterr() + assert f"{job.location}:{job.job_id}" in out + assert "Type: query" in out diff --git a/samples/snippets/noxfile.py b/samples/snippets/noxfile.py index 3bbef5d54..20cdfc620 100644 --- a/samples/snippets/noxfile.py +++ b/samples/snippets/noxfile.py @@ -187,6 +187,7 @@ def _session_tests( ) -> None: # check for presence of tests test_list = glob.glob("*_test.py") + glob.glob("test_*.py") + test_list.extend(glob.glob("tests")) if len(test_list) == 0: print("No tests found, skipping directory.") else: diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index bef333720..b47dcbc4f 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -2,10 +2,11 @@ google-cloud-bigquery-storage==2.10.1 google-auth-oauthlib==0.4.6 grpcio==1.43.0 ipython==7.16.1; python_version < '3.7' -ipython==7.29.0; python_version >= '3.7' +ipython==7.29.0; python_version == '3.7' +ipython==8.0.0; python_version >= '3.8' matplotlib==3.3.4; python_version < '3.7' -matplotlib==3.4.1; python_version >= '3.7' +matplotlib==3.5.1; python_version >= '3.7' pandas==1.1.5; python_version < '3.7' -pandas==1.3.4; python_version >= '3.7' +pandas==1.3.5; python_version >= '3.7' pyarrow==6.0.1 pytz==2021.3 diff --git a/tests/unit/test_dbapi_connection.py b/tests/unit/test_dbapi_connection.py index 0576cad38..11a268c68 100644 --- a/tests/unit/test_dbapi_connection.py +++ b/tests/unit/test_dbapi_connection.py @@ -219,8 +219,14 @@ def test_does_not_keep_cursor_instances_alive(self): # Connections should not hold strong references to the Cursor instances # they created, unnecessarily keeping them alive. gc.collect() - cursors = [obj for obj in gc.get_objects() if isinstance(obj, Cursor)] - self.assertEqual(len(cursors), 2) + cursor_count = 0 + for obj in gc.get_objects(): + try: + if isinstance(obj, Cursor): + cursor_count += 1 + except ReferenceError: # pragma: NO COVER + pass + self.assertEqual(cursor_count, 2) def test_commit(self): connection = self._make_one(client=self._mock_client()) diff --git a/tests/unit/test_magics.py b/tests/unit/test_magics.py index e18d04d64..2801768f8 100644 --- a/tests/unit/test_magics.py +++ b/tests/unit/test_magics.py @@ -1217,6 +1217,64 @@ def test_bigquery_magic_w_maximum_bytes_billed_w_context_setter(): assert sent_config["maximumBytesBilled"] == "10203" +@pytest.mark.usefixtures("ipython_interactive") +@pytest.mark.skipif(pandas is None, reason="Requires `pandas`") +def test_bigquery_magic_with_no_query_cache(monkeypatch): + ip = IPython.get_ipython() + ip.extension_manager.load_extension("google.cloud.bigquery") + conn = make_connection() + monkeypatch.setattr(magics.context, "_connection", conn) + monkeypatch.setattr(magics.context, "project", "project-from-context") + + # --no_query_cache option should override context. + monkeypatch.setattr( + magics.context.default_query_job_config, "use_query_cache", True + ) + + ip.run_cell_magic("bigquery", "--no_query_cache", QUERY_STRING) + + conn.api_request.assert_called_with( + method="POST", + path="/projects/project-from-context/jobs", + data=mock.ANY, + timeout=DEFAULT_TIMEOUT, + ) + jobs_insert_call = [ + call + for call in conn.api_request.call_args_list + if call[1]["path"] == "/projects/project-from-context/jobs" + ][0] + assert not jobs_insert_call[1]["data"]["configuration"]["query"]["useQueryCache"] + + +@pytest.mark.usefixtures("ipython_interactive") +@pytest.mark.skipif(pandas is None, reason="Requires `pandas`") +def test_context_with_no_query_cache_from_context(monkeypatch): + ip = IPython.get_ipython() + ip.extension_manager.load_extension("google.cloud.bigquery") + conn = make_connection() + monkeypatch.setattr(magics.context, "_connection", conn) + monkeypatch.setattr(magics.context, "project", "project-from-context") + monkeypatch.setattr( + magics.context.default_query_job_config, "use_query_cache", False + ) + + ip.run_cell_magic("bigquery", "", QUERY_STRING) + + conn.api_request.assert_called_with( + method="POST", + path="/projects/project-from-context/jobs", + data=mock.ANY, + timeout=DEFAULT_TIMEOUT, + ) + jobs_insert_call = [ + call + for call in conn.api_request.call_args_list + if call[1]["path"] == "/projects/project-from-context/jobs" + ][0] + assert not jobs_insert_call[1]["data"]["configuration"]["query"]["useQueryCache"] + + @pytest.mark.usefixtures("ipython_interactive") @pytest.mark.skipif(pandas is None, reason="Requires `pandas`") def test_bigquery_magic_w_progress_bar_type_w_context_setter(monkeypatch):