From 144cbf4ec8f0ba0199901d2128958a68b3b6f7fc Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Fri, 14 Jan 2022 21:43:58 +0100 Subject: [PATCH 01/13] chore(deps): update all dependencies (#1101) * chore(deps): update all dependencies * test: limit deps versions by python * remove pyproj, as it is only a transitive dependency Co-authored-by: Tim Swast Co-authored-by: Anthonios Partheniou --- samples/geography/requirements.txt | 14 ++++++-------- samples/magics/requirements.txt | 7 ++++--- samples/snippets/requirements.txt | 7 ++++--- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/samples/geography/requirements.txt b/samples/geography/requirements.txt index ca7e38f84..b07ba50bf 100644 --- a/samples/geography/requirements.txt +++ b/samples/geography/requirements.txt @@ -5,12 +5,12 @@ charset-normalizer==2.0.10 click==8.0.3 click-plugins==1.1.1 cligj==0.7.2 -dataclasses==0.6; python_version < '3.7' +dataclasses==0.8; python_version < '3.7' Fiona==1.8.20 geojson==2.5.0 geopandas==0.9.0; python_version < '3.7' geopandas==0.10.2; python_version >= '3.7' -google-api-core==2.3.2 +google-api-core==2.4.0 google-auth==2.3.3 google-cloud-bigquery==2.31.0 google-cloud-bigquery-storage==2.10.1 @@ -20,21 +20,19 @@ google-resumable-media==2.1.0 googleapis-common-protos==1.54.0 grpcio==1.43.0 idna==3.3 -libcst==0.3.23 +libcst==0.4.0 munch==2.5.0 mypy-extensions==0.4.3 packaging==21.3 pandas==1.1.5; python_version < '3.7' -pandas==1.3.4; python_version >= '3.7' +pandas==1.3.5; python_version >= '3.7' proto-plus==1.19.8 -protobuf==3.19.1 +protobuf==3.19.3 pyarrow==6.0.1 pyasn1==0.4.8 pyasn1-modules==0.2.8 pycparser==2.21 pyparsing==3.0.6 -pyproj==3.0.1; python_version < "3.7" -pyproj==3.1.0; python_version > "3.6" python-dateutil==2.8.2 pytz==2021.3 PyYAML==6.0 @@ -44,4 +42,4 @@ Shapely==1.8.0 six==1.16.0 typing-extensions==4.0.1 typing-inspect==0.7.1 -urllib3==1.26.7 +urllib3==1.26.8 diff --git a/samples/magics/requirements.txt b/samples/magics/requirements.txt index f4337e8fb..b47dcbc4f 100644 --- a/samples/magics/requirements.txt +++ b/samples/magics/requirements.txt @@ -2,10 +2,11 @@ google-cloud-bigquery-storage==2.10.1 google-auth-oauthlib==0.4.6 grpcio==1.43.0 ipython==7.16.1; python_version < '3.7' -ipython==7.29.0; python_version >= '3.7' +ipython==7.29.0; python_version == '3.7' +ipython==8.0.0; python_version >= '3.8' matplotlib==3.3.4; python_version < '3.7' -matplotlib==3.5.0rc1; python_version >= '3.7' +matplotlib==3.5.1; python_version >= '3.7' pandas==1.1.5; python_version < '3.7' -pandas==1.3.4; python_version >= '3.7' +pandas==1.3.5; python_version >= '3.7' pyarrow==6.0.1 pytz==2021.3 diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index bef333720..b47dcbc4f 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -2,10 +2,11 @@ google-cloud-bigquery-storage==2.10.1 google-auth-oauthlib==0.4.6 grpcio==1.43.0 ipython==7.16.1; python_version < '3.7' -ipython==7.29.0; python_version >= '3.7' +ipython==7.29.0; python_version == '3.7' +ipython==8.0.0; python_version >= '3.8' matplotlib==3.3.4; python_version < '3.7' -matplotlib==3.4.1; python_version >= '3.7' +matplotlib==3.5.1; python_version >= '3.7' pandas==1.1.5; python_version < '3.7' -pandas==1.3.4; python_version >= '3.7' +pandas==1.3.5; python_version >= '3.7' pyarrow==6.0.1 pytz==2021.3 From 51aced2e4354a9997ba7a9ebf7e6c72a03e35f6b Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Fri, 14 Jan 2022 16:26:41 -0500 Subject: [PATCH 02/13] chore(python): update release.sh to use keystore (#1111) Source-Link: https://github.com/googleapis/synthtool/commit/69fda12e2994f0b595a397e8bb6e3e9f380524eb Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:ae600f36b6bc972b368367b6f83a1d91ec2c82a4a116b383d67d547c56fe6de3 Co-authored-by: Owl Bot Co-authored-by: Tim Swast --- .github/.OwlBot.lock.yaml | 2 +- .kokoro/release.sh | 2 +- .kokoro/release/common.cfg | 12 +++++++++++- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index ff5126c18..eecb84c21 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -1,3 +1,3 @@ docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:dfa9b663b32de8b5b327e32c1da665a80de48876558dd58091d8160c60ad7355 + digest: sha256:ae600f36b6bc972b368367b6f83a1d91ec2c82a4a116b383d67d547c56fe6de3 diff --git a/.kokoro/release.sh b/.kokoro/release.sh index 3abba6e06..b030caeef 100755 --- a/.kokoro/release.sh +++ b/.kokoro/release.sh @@ -26,7 +26,7 @@ python3 -m pip install --upgrade twine wheel setuptools export PYTHONUNBUFFERED=1 # Move into the package, build the distribution and upload. -TWINE_PASSWORD=$(cat "${KOKORO_GFILE_DIR}/secret_manager/google-cloud-pypi-token") +TWINE_PASSWORD=$(cat "${KOKORO_KEYSTORE_DIR}/73713_google-cloud-pypi-token-keystore-1") cd github/python-bigquery python3 setup.py sdist bdist_wheel twine upload --username __token__ --password "${TWINE_PASSWORD}" dist/* diff --git a/.kokoro/release/common.cfg b/.kokoro/release/common.cfg index 922d7fe50..6ae81b743 100644 --- a/.kokoro/release/common.cfg +++ b/.kokoro/release/common.cfg @@ -23,8 +23,18 @@ env_vars: { value: "github/python-bigquery/.kokoro/release.sh" } +# Fetch PyPI password +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73713 + keyname: "google-cloud-pypi-token-keystore-1" + } + } +} + # Tokens needed to report release status back to GitHub env_vars: { key: "SECRET_MANAGER_KEYS" - value: "releasetool-publish-reporter-app,releasetool-publish-reporter-googleapis-installation,releasetool-publish-reporter-pem,google-cloud-pypi-token" + value: "releasetool-publish-reporter-app,releasetool-publish-reporter-googleapis-installation,releasetool-publish-reporter-pem" } From f54d81fc5b4bb834fe5054de369f7158fff9bb25 Mon Sep 17 00:00:00 2001 From: Matthew Suozzo Date: Tue, 18 Jan 2022 11:21:23 -0500 Subject: [PATCH 03/13] test: make GC check resilient to broken weakrefs (#1099) * fix: Make gc check resilient to broken weakrefs In some python deployments (I believe macOS is one), weakref objects aren't guaranteed to exist across gc collections. The proposed check pattern is also used in tensorflow [0] which ran into similar issues a few years ago. [0]: https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/framework/errors_test.py#L33 * fix: add pragma for coverage * Fix whitespace --- tests/unit/test_dbapi_connection.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_dbapi_connection.py b/tests/unit/test_dbapi_connection.py index 0576cad38..11a268c68 100644 --- a/tests/unit/test_dbapi_connection.py +++ b/tests/unit/test_dbapi_connection.py @@ -219,8 +219,14 @@ def test_does_not_keep_cursor_instances_alive(self): # Connections should not hold strong references to the Cursor instances # they created, unnecessarily keeping them alive. gc.collect() - cursors = [obj for obj in gc.get_objects() if isinstance(obj, Cursor)] - self.assertEqual(len(cursors), 2) + cursor_count = 0 + for obj in gc.get_objects(): + try: + if isinstance(obj, Cursor): + cursor_count += 1 + except ReferenceError: # pragma: NO COVER + pass + self.assertEqual(cursor_count, 2) def test_commit(self): connection = self._make_one(client=self._mock_client()) From fa8aea03879363f4d12ef0c2557047b56ae08714 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Tue, 18 Jan 2022 20:24:46 -0500 Subject: [PATCH 04/13] chore(python): Noxfile recognizes that tests can live in a folder (#1115) Source-Link: https://github.com/googleapis/synthtool/commit/4760d8dce1351d93658cb11d02a1b7ceb23ae5d7 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:f0e4b51deef56bed74d3e2359c583fc104a8d6367da3984fc5c66938db738828 Co-authored-by: Owl Bot --- .github/.OwlBot.lock.yaml | 2 +- samples/geography/noxfile.py | 1 + samples/magics/noxfile.py | 1 + samples/snippets/noxfile.py | 1 + 4 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index eecb84c21..52d79c11f 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -1,3 +1,3 @@ docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:ae600f36b6bc972b368367b6f83a1d91ec2c82a4a116b383d67d547c56fe6de3 + digest: sha256:f0e4b51deef56bed74d3e2359c583fc104a8d6367da3984fc5c66938db738828 diff --git a/samples/geography/noxfile.py b/samples/geography/noxfile.py index 3bbef5d54..20cdfc620 100644 --- a/samples/geography/noxfile.py +++ b/samples/geography/noxfile.py @@ -187,6 +187,7 @@ def _session_tests( ) -> None: # check for presence of tests test_list = glob.glob("*_test.py") + glob.glob("test_*.py") + test_list.extend(glob.glob("tests")) if len(test_list) == 0: print("No tests found, skipping directory.") else: diff --git a/samples/magics/noxfile.py b/samples/magics/noxfile.py index 3bbef5d54..20cdfc620 100644 --- a/samples/magics/noxfile.py +++ b/samples/magics/noxfile.py @@ -187,6 +187,7 @@ def _session_tests( ) -> None: # check for presence of tests test_list = glob.glob("*_test.py") + glob.glob("test_*.py") + test_list.extend(glob.glob("tests")) if len(test_list) == 0: print("No tests found, skipping directory.") else: diff --git a/samples/snippets/noxfile.py b/samples/snippets/noxfile.py index 3bbef5d54..20cdfc620 100644 --- a/samples/snippets/noxfile.py +++ b/samples/snippets/noxfile.py @@ -187,6 +187,7 @@ def _session_tests( ) -> None: # check for presence of tests test_list = glob.glob("*_test.py") + glob.glob("test_*.py") + test_list.extend(glob.glob("tests")) if len(test_list) == 0: print("No tests found, skipping directory.") else: From 970c2a40b1bda85c897ddf091aaeb6a3b326c249 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Fri, 21 Jan 2022 20:12:50 -0500 Subject: [PATCH 05/13] chore(python): exclude templated GH action workflows (#1116) * ci(python): run lint / unit tests / docs as GH actions Source-Link: https://github.com/googleapis/synthtool/commit/57be0cdb0b94e1669cee0ca38d790de1dfdbcd44 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:ed1f9983d5a935a89fe8085e8bb97d94e41015252c5b6c9771257cf8624367e6 * exclude templated GH action workflows Co-authored-by: Owl Bot Co-authored-by: Anthonios Partheniou --- .github/.OwlBot.lock.yaml | 15 ++++++++++++++- owlbot.py | 1 + 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 52d79c11f..8cb43804d 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -1,3 +1,16 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:f0e4b51deef56bed74d3e2359c583fc104a8d6367da3984fc5c66938db738828 + digest: sha256:ed1f9983d5a935a89fe8085e8bb97d94e41015252c5b6c9771257cf8624367e6 diff --git a/owlbot.py b/owlbot.py index e6f36905b..095759d48 100644 --- a/owlbot.py +++ b/owlbot.py @@ -116,6 +116,7 @@ # Include custom SNIPPETS_TESTS job for performance. # https://github.com/googleapis/python-bigquery/issues/191 ".kokoro/presubmit/presubmit.cfg", + ".github/workflows", # exclude gh actions as credentials are needed for tests ], ) From aee4e61f9ae9cbb50db4d07f8c2fec9cb5e0006a Mon Sep 17 00:00:00 2001 From: Bu Sun Kim <8822365+busunkim96@users.noreply.github.com> Date: Mon, 24 Jan 2022 13:08:06 -0700 Subject: [PATCH 06/13] chore: make samples 3.6 check optional (#1120) --- .github/sync-repo-settings.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/sync-repo-settings.yaml b/.github/sync-repo-settings.yaml index 01affbae5..73cc3bcef 100644 --- a/.github/sync-repo-settings.yaml +++ b/.github/sync-repo-settings.yaml @@ -14,7 +14,6 @@ branchProtectionRules: - 'Kokoro snippets-3.8' - 'cla/google' - 'Samples - Lint' - - 'Samples - Python 3.6' - 'Samples - Python 3.7' - 'Samples - Python 3.8' - pattern: v3 @@ -26,6 +25,5 @@ branchProtectionRules: - 'Kokoro snippets-3.8' - 'cla/google' - 'Samples - Lint' - - 'Samples - Python 3.6' - 'Samples - Python 3.7' - 'Samples - Python 3.8' From 8712b6f4f6c91b0f5aca3de98efe81deb2c7262f Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Thu, 3 Feb 2022 20:18:01 -0500 Subject: [PATCH 07/13] chore: use gapic-generator-python 0.62.1 (#1127) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit committer: parthea PiperOrigin-RevId: 425964861 Source-Link: https://github.com/googleapis/googleapis/commit/84b1a5a4f6fb2d04905be58e586b8a7a4310a8cf Source-Link: https://github.com/googleapis/googleapis-gen/commit/4fb761bbd8506ac156f49bac5f18306aa8eb3aa8 Copy-Tag: eyJwIjoiLmdpdGh1Yi8uT3dsQm90LnlhbWwiLCJoIjoiNGZiNzYxYmJkODUwNmFjMTU2ZjQ5YmFjNWYxODMwNmFhOGViM2FhOCJ9 * 🦉 Updates from OwlBot See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md Co-authored-by: Owl Bot --- google/cloud/bigquery_v2/types/model.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/google/cloud/bigquery_v2/types/model.py b/google/cloud/bigquery_v2/types/model.py index 440bc0805..84188e40c 100644 --- a/google/cloud/bigquery_v2/types/model.py +++ b/google/cloud/bigquery_v2/types/model.py @@ -341,8 +341,8 @@ class AggregateClassificationMetrics(proto.Message): threshold. f1_score (google.protobuf.wrappers_pb2.DoubleValue): The F1 score is an average of recall and - precision. For multiclass this is a macro- - averaged metric. + precision. For multiclass this is a + macro-averaged metric. log_loss (google.protobuf.wrappers_pb2.DoubleValue): Logarithmic Loss. For multiclass this is a macro-averaged metric. From 8edc10d019bd96defebc4f92a47774901e9b956f Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Mon, 14 Feb 2022 16:02:30 -0600 Subject: [PATCH 08/13] docs: show common job properties in `get_job` and `cancel_job` samples (#1137) * docs: show common job properties in `get_job` and `cancel_job` samples * flake8 --- docs/snippets.py | 1 + samples/snippets/manage_job_cancel.py | 26 ++++++++++++++++++ samples/snippets/manage_job_get.py | 33 +++++++++++++++++++++++ samples/snippets/manage_job_test.py | 39 +++++++++++++++++++++++++++ 4 files changed, 99 insertions(+) create mode 100644 samples/snippets/manage_job_cancel.py create mode 100644 samples/snippets/manage_job_get.py create mode 100644 samples/snippets/manage_job_test.py diff --git a/docs/snippets.py b/docs/snippets.py index c62001fc0..499285eeb 100644 --- a/docs/snippets.py +++ b/docs/snippets.py @@ -757,6 +757,7 @@ def test_client_query_total_rows(client, capsys): def test_manage_job(client): + # TODO(b/199162556): delete after migrating docs sql = """ SELECT corpus FROM `bigquery-public-data.samples.shakespeare` diff --git a/samples/snippets/manage_job_cancel.py b/samples/snippets/manage_job_cancel.py new file mode 100644 index 000000000..3e0fc5218 --- /dev/null +++ b/samples/snippets/manage_job_cancel.py @@ -0,0 +1,26 @@ +# Copyright 2016-2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START bigquery_cancel_job] +from google.cloud import bigquery + + +def cancel_job( + client: bigquery.Client, location: str = "us", job_id: str = "abcd-efgh-ijkl-mnop", +): + job = client.cancel_job(job_id, location=location) + print(f"{job.location}:{job.job_id} cancelled") + + +# [END bigquery_cancel_job] diff --git a/samples/snippets/manage_job_get.py b/samples/snippets/manage_job_get.py new file mode 100644 index 000000000..256d79e5b --- /dev/null +++ b/samples/snippets/manage_job_get.py @@ -0,0 +1,33 @@ +# Copyright 2016-2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START bigquery_get_job] +from google.cloud import bigquery + + +def get_job( + client: bigquery.Client, location: str = "us", job_id: str = "abcd-efgh-ijkl-mnop", +): + job = client.get_job(job_id, location=location) + + # All job classes have "location" and "job_id" string properties. + # Use these properties for job operations such as "cancel_job" and + # "delete_job". + print(f"{job.location}:{job.job_id}") + print(f"Type: {job.job_type}") + print(f"State: {job.state}") + print(f"Created: {job.created.isoformat()}") + + +# [END bigquery_get_job] diff --git a/samples/snippets/manage_job_test.py b/samples/snippets/manage_job_test.py new file mode 100644 index 000000000..745b7bbbe --- /dev/null +++ b/samples/snippets/manage_job_test.py @@ -0,0 +1,39 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from google.cloud import bigquery +import pytest + +import manage_job_cancel +import manage_job_get + + +def test_manage_job(capsys: pytest.CaptureFixture): + client = bigquery.Client() + sql = """ + SELECT corpus + FROM `bigquery-public-data.samples.shakespeare` + GROUP BY corpus; + """ + location = "us" + job = client.query(sql, location=location) + + manage_job_cancel.cancel_job(client, location=location, job_id=job.job_id) + out, _ = capsys.readouterr() + assert f"{job.location}:{job.job_id} cancelled" in out + + manage_job_get.get_job(client, location=location, job_id=job.job_id) + out, _ = capsys.readouterr() + assert f"{job.location}:{job.job_id}" in out + assert "Type: query" in out From 18d958062721d6be81e7bd7a5bd66f277344a864 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Mon, 14 Feb 2022 17:19:09 -0600 Subject: [PATCH 09/13] =?UTF-8?q?docs:=20reference=20BigQuery=20REST=20API?= =?UTF-8?q?=20defaults=20in=20`LoadJobConfig`=20descrip=E2=80=A6=20(#1132)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit …tion Thank you for opening a Pull Request! Before submitting your PR, there are a few things you can do to make sure it goes smoothly: - [ ] Make sure to open an issue as a [bug/issue](https://github.com/googleapis/python-bigquery/issues/new/choose) before writing your code! That way we can discuss the change, evaluate designs, and agree on the general idea - [ ] Ensure the tests and linter pass - [ ] Code coverage does not decrease (if any source code was changed) - [ ] Appropriate docs were updated (if necessary) Fixes internal issue 213442215 🦕 --- google/cloud/bigquery/job/load.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/google/cloud/bigquery/job/load.py b/google/cloud/bigquery/job/load.py index b12c3e621..2d68f7f71 100644 --- a/google/cloud/bigquery/job/load.py +++ b/google/cloud/bigquery/job/load.py @@ -33,9 +33,21 @@ class LoadJobConfig(_JobConfig): """Configuration options for load jobs. - All properties in this class are optional. Values which are :data:`None` -> - server defaults. Set properties on the constructed configuration by using - the property name as the name of a keyword argument. + Set properties on the constructed configuration by using the property name + as the name of a keyword argument. Values which are unset or :data:`None` + use the BigQuery REST API default values. See the `BigQuery REST API + reference documentation + `_ + for a list of default values. + + Required options differ based on the + :attr:`~google.cloud.bigquery.job.LoadJobConfig.source_format` value. + For example, the BigQuery API's default value for + :attr:`~google.cloud.bigquery.job.LoadJobConfig.source_format` is ``"CSV"``. + When loading a CSV file, either + :attr:`~google.cloud.bigquery.job.LoadJobConfig.schema` must be set or + :attr:`~google.cloud.bigquery.job.LoadJobConfig.autodetect` must be set to + :data:`True`. """ def __init__(self, **kwargs): From 1c7dbdd21a4399dd26c699f7ba55c6019a3744d2 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Wed, 16 Feb 2022 13:51:32 -0600 Subject: [PATCH 10/13] chore: remove unused code snippets (#1139) --- docs/snippets.py | 40 ---------------------------------------- 1 file changed, 40 deletions(-) diff --git a/docs/snippets.py b/docs/snippets.py index 499285eeb..f67823249 100644 --- a/docs/snippets.py +++ b/docs/snippets.py @@ -756,46 +756,6 @@ def test_client_query_total_rows(client, capsys): assert "Got 100 rows." in out -def test_manage_job(client): - # TODO(b/199162556): delete after migrating docs - sql = """ - SELECT corpus - FROM `bigquery-public-data.samples.shakespeare` - GROUP BY corpus; - """ - location = "us" - job = client.query(sql, location=location) - job_id = job.job_id - - # [START bigquery_cancel_job] - # TODO(developer): Uncomment the lines below and replace with your values. - # from google.cloud import bigquery - # client = bigquery.Client() - # job_id = 'bq-job-123x456-123y123z123c' # replace with your job ID - # location = 'us' # replace with your location - - job = client.cancel_job(job_id, location=location) - # [END bigquery_cancel_job] - - # [START bigquery_get_job] - # TODO(developer): Uncomment the lines below and replace with your values. - # from google.cloud import bigquery - # client = bigquery.Client() - # job_id = 'bq-job-123x456-123y123z123c' # replace with your job ID - # location = 'us' # replace with your location - - job = client.get_job(job_id, location=location) # API request - - # Print selected job properties - print("Details for job {} running in {}:".format(job_id, location)) - print( - "\tType: {}\n\tState: {}\n\tCreated: {}".format( - job.job_type, job.state, job.created - ) - ) - # [END bigquery_get_job] - - def test_query_external_gcs_permanent_table(client, to_delete): dataset_id = "query_external_gcs_{}".format(_millis()) project = client.project From f5daa9b41377a58cb3220bb2ab7c72adc6462196 Mon Sep 17 00:00:00 2001 From: Erroration2022 <99040129+Erroration2022@users.noreply.github.com> Date: Wed, 16 Feb 2022 12:20:47 -0800 Subject: [PATCH 11/13] fix: return 403 when VPC-SC violation happens (#1131) This is fixing the issue where VPC-SC violation is not returning 403. Error message map does not recognize VPCSC policy violation error and will default to return an internal server error. Co-authored-by: Tim Swast --- google/cloud/bigquery/job/base.py | 1 + 1 file changed, 1 insertion(+) diff --git a/google/cloud/bigquery/job/base.py b/google/cloud/bigquery/job/base.py index 97acab5d2..86701e295 100644 --- a/google/cloud/bigquery/job/base.py +++ b/google/cloud/bigquery/job/base.py @@ -45,6 +45,7 @@ "invalidQuery": http.client.BAD_REQUEST, "notFound": http.client.NOT_FOUND, "notImplemented": http.client.NOT_IMPLEMENTED, + "policyViolation": http.client.FORBIDDEN, "quotaExceeded": http.client.FORBIDDEN, "rateLimitExceeded": http.client.FORBIDDEN, "resourceInUse": http.client.BAD_REQUEST, From 7dd30af41b8a595b96176c964ba14aa41645ef0d Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Wed, 16 Feb 2022 17:07:47 -0600 Subject: [PATCH 12/13] feat: add `--no_query_cache` option to `%%bigquery` magics to disable query cache (#1141) --- google/cloud/bigquery/magics/magics.py | 12 ++++++ tests/unit/test_magics.py | 58 ++++++++++++++++++++++++++ 2 files changed, 70 insertions(+) diff --git a/google/cloud/bigquery/magics/magics.py b/google/cloud/bigquery/magics/magics.py index 5af0a3b51..7b4d584fb 100644 --- a/google/cloud/bigquery/magics/magics.py +++ b/google/cloud/bigquery/magics/magics.py @@ -35,6 +35,8 @@ A dataset and table to store the query results. If table does not exists, it will be created. If table already exists, its data will be overwritten. Variable should be in a format .. + * ``--no_query_cache`` (Optional[line argument]): + Do not use cached query results. * ``--project `` (Optional[line argument]): Project to use for running the query. Defaults to the context :attr:`~google.cloud.bigquery.magics.Context.project`. @@ -442,6 +444,12 @@ def _create_dataset_if_necessary(client, dataset_id): "this option's value in the context bqstorage_client_options." ), ) +@magic_arguments.argument( + "--no_query_cache", + action="store_true", + default=False, + help=("Do not use cached query results."), +) @magic_arguments.argument( "--use_bqstorage_api", action="store_true", @@ -642,6 +650,10 @@ def _cell_magic(line, query): job_config.use_legacy_sql = args.use_legacy_sql job_config.dry_run = args.dry_run + # Don't override context job config unless --no_query_cache is explicitly set. + if args.no_query_cache: + job_config.use_query_cache = False + if args.destination_table: split = args.destination_table.split(".") if len(split) != 2: diff --git a/tests/unit/test_magics.py b/tests/unit/test_magics.py index e18d04d64..2801768f8 100644 --- a/tests/unit/test_magics.py +++ b/tests/unit/test_magics.py @@ -1217,6 +1217,64 @@ def test_bigquery_magic_w_maximum_bytes_billed_w_context_setter(): assert sent_config["maximumBytesBilled"] == "10203" +@pytest.mark.usefixtures("ipython_interactive") +@pytest.mark.skipif(pandas is None, reason="Requires `pandas`") +def test_bigquery_magic_with_no_query_cache(monkeypatch): + ip = IPython.get_ipython() + ip.extension_manager.load_extension("google.cloud.bigquery") + conn = make_connection() + monkeypatch.setattr(magics.context, "_connection", conn) + monkeypatch.setattr(magics.context, "project", "project-from-context") + + # --no_query_cache option should override context. + monkeypatch.setattr( + magics.context.default_query_job_config, "use_query_cache", True + ) + + ip.run_cell_magic("bigquery", "--no_query_cache", QUERY_STRING) + + conn.api_request.assert_called_with( + method="POST", + path="/projects/project-from-context/jobs", + data=mock.ANY, + timeout=DEFAULT_TIMEOUT, + ) + jobs_insert_call = [ + call + for call in conn.api_request.call_args_list + if call[1]["path"] == "/projects/project-from-context/jobs" + ][0] + assert not jobs_insert_call[1]["data"]["configuration"]["query"]["useQueryCache"] + + +@pytest.mark.usefixtures("ipython_interactive") +@pytest.mark.skipif(pandas is None, reason="Requires `pandas`") +def test_context_with_no_query_cache_from_context(monkeypatch): + ip = IPython.get_ipython() + ip.extension_manager.load_extension("google.cloud.bigquery") + conn = make_connection() + monkeypatch.setattr(magics.context, "_connection", conn) + monkeypatch.setattr(magics.context, "project", "project-from-context") + monkeypatch.setattr( + magics.context.default_query_job_config, "use_query_cache", False + ) + + ip.run_cell_magic("bigquery", "", QUERY_STRING) + + conn.api_request.assert_called_with( + method="POST", + path="/projects/project-from-context/jobs", + data=mock.ANY, + timeout=DEFAULT_TIMEOUT, + ) + jobs_insert_call = [ + call + for call in conn.api_request.call_args_list + if call[1]["path"] == "/projects/project-from-context/jobs" + ][0] + assert not jobs_insert_call[1]["data"]["configuration"]["query"]["useQueryCache"] + + @pytest.mark.usefixtures("ipython_interactive") @pytest.mark.skipif(pandas is None, reason="Requires `pandas`") def test_bigquery_magic_w_progress_bar_type_w_context_setter(monkeypatch): From 39ade3908c4d311cd817cda0e6ce83d7bc563de0 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Thu, 17 Feb 2022 08:08:58 -0600 Subject: [PATCH 13/13] chore(main): release 2.33.0 (#1138) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- CHANGELOG.md | 18 ++++++++++++++++++ google/cloud/bigquery/version.py | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e69fa621..e83cb9788 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,24 @@ [1]: https://pypi.org/project/google-cloud-bigquery/#history +## [2.33.0](https://github.com/googleapis/python-bigquery/compare/v2.32.0...v2.33.0) (2022-02-16) + + +### Features + +* add `--no_query_cache` option to `%%bigquery` magics to disable query cache ([#1141](https://github.com/googleapis/python-bigquery/issues/1141)) ([7dd30af](https://github.com/googleapis/python-bigquery/commit/7dd30af41b8a595b96176c964ba14aa41645ef0d)) + + +### Bug Fixes + +* return 403 when VPC-SC violation happens ([#1131](https://github.com/googleapis/python-bigquery/issues/1131)) ([f5daa9b](https://github.com/googleapis/python-bigquery/commit/f5daa9b41377a58cb3220bb2ab7c72adc6462196)) + + +### Documentation + +* reference BigQuery REST API defaults in `LoadJobConfig` descrip… ([#1132](https://github.com/googleapis/python-bigquery/issues/1132)) ([18d9580](https://github.com/googleapis/python-bigquery/commit/18d958062721d6be81e7bd7a5bd66f277344a864)) +* show common job properties in `get_job` and `cancel_job` samples ([#1137](https://github.com/googleapis/python-bigquery/issues/1137)) ([8edc10d](https://github.com/googleapis/python-bigquery/commit/8edc10d019bd96defebc4f92a47774901e9b956f)) + ## [2.32.0](https://github.com/googleapis/python-bigquery/compare/v2.31.0...v2.32.0) (2022-01-12) diff --git a/google/cloud/bigquery/version.py b/google/cloud/bigquery/version.py index b8c5af9a2..17811b65f 100644 --- a/google/cloud/bigquery/version.py +++ b/google/cloud/bigquery/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.32.0" +__version__ = "2.33.0"