From d64f5b682854a2293244426316890df4ab1e079e Mon Sep 17 00:00:00 2001 From: Steffany Brown <30247553+steffnay@users.noreply.github.com> Date: Wed, 27 Oct 2021 14:09:08 -0700 Subject: [PATCH 1/6] docs(samples): add create external table with hive partitioning (#1033) * docs(samples): add create table hive partitioning sample * refactor --- .../create_table_external_hive_partitioned.py | 73 +++++++++++++++++++ ...te_table_external_hive_partitioned_test.py | 31 ++++++++ 2 files changed, 104 insertions(+) create mode 100644 samples/snippets/create_table_external_hive_partitioned.py create mode 100644 samples/snippets/create_table_external_hive_partitioned_test.py diff --git a/samples/snippets/create_table_external_hive_partitioned.py b/samples/snippets/create_table_external_hive_partitioned.py new file mode 100644 index 000000000..2ff8a2220 --- /dev/null +++ b/samples/snippets/create_table_external_hive_partitioned.py @@ -0,0 +1,73 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def create_table_external_hive_partitioned(table_id: str): + original_table_id = table_id + # [START bigquery_create_table_external_hivepartitioned] + # Demonstrates creating an external table with hive partitioning. + + # TODO(developer): Set table_id to the ID of the table to create. + table_id = "your-project.your_dataset.your_table_name" + + # TODO(developer): Set source uri. + # Example file: + # gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/dt=2020-11-15/file1.parquet + uri = "gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/*" + + # TODO(developer): Set source uri prefix. + source_uri_prefix = ( + "gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/" + ) + + # [END bigquery_create_table_external_hivepartitioned] + table_id = original_table_id + # [START bigquery_create_table_external_hivepartitioned] + from google.cloud import bigquery + + # Construct a BigQuery client object. + client = bigquery.Client() + + # Configure the external data source. + external_config = bigquery.ExternalConfig("PARQUET") + external_config.source_uris = [uri] + external_config.autodetect = True + + # Configure partitioning options. + hive_partitioning_opts = bigquery.external_config.HivePartitioningOptions() + + # The layout of the files in here is compatible with the layout requirements for hive partitioning, + # so we can add an optional Hive partitioning configuration to leverage the object paths for deriving + # partitioning column information. + + # For more information on how partitions are extracted, see: + # https://cloud.google.com/bigquery/docs/hive-partitioned-queries-gcs + + # We have a "/dt=YYYY-MM-DD/" path component in our example files as documented above. + # Autolayout will expose this as a column named "dt" of type DATE. + hive_partitioning_opts.mode = "AUTO" + hive_partitioning_opts.require_partition_filter = True + hive_partitioning_opts.source_uri_prefix = source_uri_prefix + + external_config.hive_partitioning = hive_partitioning_opts + + table = bigquery.Table(table_id) + table.external_data_configuration = external_config + + table = client.create_table(table) # Make an API request. + print( + "Created table {}.{}.{}".format(table.project, table.dataset_id, table.table_id) + ) + # [END bigquery_create_table_external_hivepartitioned] + return table diff --git a/samples/snippets/create_table_external_hive_partitioned_test.py b/samples/snippets/create_table_external_hive_partitioned_test.py new file mode 100644 index 000000000..c3cdddb55 --- /dev/null +++ b/samples/snippets/create_table_external_hive_partitioned_test.py @@ -0,0 +1,31 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import create_table_external_hive_partitioned + + +def test_create_table_external_hive_partitioned(capsys, random_table_id): + table = create_table_external_hive_partitioned.create_table_external_hive_partitioned( + random_table_id + ) + + out, _ = capsys.readouterr() + hive_partioning = table.external_data_configuration.hive_partitioning + assert "Created table {}".format(random_table_id) in out + assert ( + hive_partioning.source_uri_prefix + == "gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/" + ) + assert hive_partioning.require_partition_filter is True + assert hive_partioning.mode == "AUTO" From 61141ee0634024ad261d1595c95cd14a896fb87e Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Fri, 29 Oct 2021 13:30:34 -0500 Subject: [PATCH 2/6] docs: add code samples for Jupyter/IPython magics (#1013) Follow-up to https://github.com/GoogleCloudPlatform/python-docs-samples/pull/6889, which removed a BigQuery magics sample for using query parameters. Note: jupyter_tutorial_test.py is a copy of what is in the `samples/snippets` folder. Once the docs have been updated to point to this new version, we can remove that copy and remove the Jupyter/IPython depedencencies from `samples/snippets/requirements.txt`. Thank you for opening a Pull Request! Before submitting your PR, there are a few things you can do to make sure it goes smoothly: - [ ] Make sure to open an issue as a [bug/issue](https://github.com/googleapis/python-bigquery/issues/new/choose) before writing your code! That way we can discuss the change, evaluate designs, and agree on the general idea - [ ] Ensure the tests and linter pass - [ ] Code coverage does not decrease (if any source code was changed) - [ ] Appropriate docs were updated (if necessary) --- docs/magics.rst | 29 +++ google/cloud/bigquery/magics/magics.py | 66 ----- noxfile.py | 3 +- samples/magics/__init__.py | 13 + samples/magics/_helpers.py | 21 ++ samples/magics/conftest.py | 36 +++ samples/magics/noxfile.py | 266 ++++++++++++++++++++ samples/magics/query.py | 37 +++ samples/magics/query_params_scalars.py | 38 +++ samples/magics/query_params_scalars_test.py | 23 ++ samples/magics/query_test.py | 23 ++ samples/magics/requirements-test.txt | 3 + samples/magics/requirements.txt | 12 + 13 files changed, 503 insertions(+), 67 deletions(-) create mode 100644 samples/magics/__init__.py create mode 100644 samples/magics/_helpers.py create mode 100644 samples/magics/conftest.py create mode 100644 samples/magics/noxfile.py create mode 100644 samples/magics/query.py create mode 100644 samples/magics/query_params_scalars.py create mode 100644 samples/magics/query_params_scalars_test.py create mode 100644 samples/magics/query_test.py create mode 100644 samples/magics/requirements-test.txt create mode 100644 samples/magics/requirements.txt diff --git a/docs/magics.rst b/docs/magics.rst index bcaad8fa3..aa14c6bfa 100644 --- a/docs/magics.rst +++ b/docs/magics.rst @@ -1,5 +1,34 @@ IPython Magics for BigQuery =========================== +To use these magics, you must first register them. Run the ``%load_ext`` magic +in a Jupyter notebook cell. + +.. code:: + + %load_ext google.cloud.bigquery + +This makes the ``%%bigquery`` magic available. + +Code Samples +------------ + +Running a query: + +.. literalinclude:: ./samples/magics/query.py + :dedent: 4 + :start-after: [START bigquery_jupyter_query] + :end-before: [END bigquery_jupyter_query] + +Running a parameterized query: + +.. literalinclude:: ./samples/magics/query_params_scalars.py + :dedent: 4 + :start-after: [START bigquery_jupyter_query_params_scalars] + :end-before: [END bigquery_jupyter_query_params_scalars] + +API Reference +------------- + .. automodule:: google.cloud.bigquery.magics.magics :members: diff --git a/google/cloud/bigquery/magics/magics.py b/google/cloud/bigquery/magics/magics.py index d368bbeaa..ec0430518 100644 --- a/google/cloud/bigquery/magics/magics.py +++ b/google/cloud/bigquery/magics/magics.py @@ -14,15 +14,6 @@ """IPython Magics -To use these magics, you must first register them. Run the ``%load_ext`` magic -in a Jupyter notebook cell. - -.. code:: - - %load_ext google.cloud.bigquery - -This makes the ``%%bigquery`` magic available. - .. function:: %%bigquery IPython cell magic to run a query and display the result as a DataFrame @@ -85,63 +76,6 @@ .. note:: All queries run using this magic will run using the context :attr:`~google.cloud.bigquery.magics.Context.credentials`. - - Examples: - The following examples can be run in an IPython notebook after loading - the bigquery IPython extension (see ``In[1]``) and setting up - Application Default Credentials. - - .. code-block:: none - - In [1]: %load_ext google.cloud.bigquery - - In [2]: %%bigquery - ...: SELECT name, SUM(number) as count - ...: FROM `bigquery-public-data.usa_names.usa_1910_current` - ...: GROUP BY name - ...: ORDER BY count DESC - ...: LIMIT 3 - - Out[2]: name count - ...: ------------------- - ...: 0 James 4987296 - ...: 1 John 4866302 - ...: 2 Robert 4738204 - - In [3]: %%bigquery df --project my-alternate-project --verbose - ...: SELECT name, SUM(number) as count - ...: FROM `bigquery-public-data.usa_names.usa_1910_current` - ...: WHERE gender = 'F' - ...: GROUP BY name - ...: ORDER BY count DESC - ...: LIMIT 3 - Executing query with job ID: bf633912-af2c-4780-b568-5d868058632b - Query executing: 2.61s - Query complete after 2.92s - - In [4]: df - - Out[4]: name count - ...: ---------------------- - ...: 0 Mary 3736239 - ...: 1 Patricia 1568495 - ...: 2 Elizabeth 1519946 - - In [5]: %%bigquery --params {"num": 17} - ...: SELECT @num AS num - - Out[5]: num - ...: ------- - ...: 0 17 - - In [6]: params = {"num": 17} - - In [7]: %%bigquery --params $params - ...: SELECT @num AS num - - Out[7]: num - ...: ------- - ...: 0 17 """ from __future__ import print_function diff --git a/noxfile.py b/noxfile.py index d41573407..64eacaff5 100644 --- a/noxfile.py +++ b/noxfile.py @@ -186,8 +186,9 @@ def snippets(session): session.run( "py.test", "samples", - "--ignore=samples/snippets", + "--ignore=samples/magics", "--ignore=samples/geography", + "--ignore=samples/snippets", *session.posargs, ) diff --git a/samples/magics/__init__.py b/samples/magics/__init__.py new file mode 100644 index 000000000..4fbd93bb2 --- /dev/null +++ b/samples/magics/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/samples/magics/_helpers.py b/samples/magics/_helpers.py new file mode 100644 index 000000000..18a513b99 --- /dev/null +++ b/samples/magics/_helpers.py @@ -0,0 +1,21 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def strip_region_tags(sample_text): + """Remove blank lines and region tags from sample text""" + magic_lines = [ + line for line in sample_text.split("\n") if len(line) > 0 and "# [" not in line + ] + return "\n".join(magic_lines) diff --git a/samples/magics/conftest.py b/samples/magics/conftest.py new file mode 100644 index 000000000..bf8602235 --- /dev/null +++ b/samples/magics/conftest.py @@ -0,0 +1,36 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +interactiveshell = pytest.importorskip("IPython.terminal.interactiveshell") +tools = pytest.importorskip("IPython.testing.tools") + + +@pytest.fixture(scope="session") +def ipython(): + config = tools.default_config() + config.TerminalInteractiveShell.simple_prompt = True + shell = interactiveshell.TerminalInteractiveShell.instance(config=config) + return shell + + +@pytest.fixture(autouse=True) +def ipython_interactive(ipython): + """Activate IPython's builtin hooks + + for the duration of the test scope. + """ + with ipython.builtin_trap: + yield ipython diff --git a/samples/magics/noxfile.py b/samples/magics/noxfile.py new file mode 100644 index 000000000..b008613f0 --- /dev/null +++ b/samples/magics/noxfile.py @@ -0,0 +1,266 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import os +from pathlib import Path +import sys +from typing import Callable, Dict, List, Optional + +import nox + + +# WARNING - WARNING - WARNING - WARNING - WARNING +# WARNING - WARNING - WARNING - WARNING - WARNING +# DO NOT EDIT THIS FILE EVER! +# WARNING - WARNING - WARNING - WARNING - WARNING +# WARNING - WARNING - WARNING - WARNING - WARNING + +BLACK_VERSION = "black==19.10b0" + +# Copy `noxfile_config.py` to your directory and modify it instead. + +# `TEST_CONFIG` dict is a configuration hook that allows users to +# modify the test configurations. The values here should be in sync +# with `noxfile_config.py`. Users will copy `noxfile_config.py` into +# their directory and modify it. + +TEST_CONFIG = { + # You can opt out from the test for specific Python versions. + "ignored_versions": [], + # Old samples are opted out of enforcing Python type hints + # All new samples should feature them + "enforce_type_hints": False, + # An envvar key for determining the project id to use. Change it + # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a + # build specific Cloud project. You can also use your own string + # to use your own Cloud project. + "gcloud_project_env": "GOOGLE_CLOUD_PROJECT", + # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', + # If you need to use a specific version of pip, + # change pip_version_override to the string representation + # of the version number, for example, "20.2.4" + "pip_version_override": None, + # A dictionary you want to inject into your test. Don't put any + # secrets here. These values will override predefined values. + "envs": {}, +} + + +try: + # Ensure we can import noxfile_config in the project's directory. + sys.path.append(".") + from noxfile_config import TEST_CONFIG_OVERRIDE +except ImportError as e: + print("No user noxfile_config found: detail: {}".format(e)) + TEST_CONFIG_OVERRIDE = {} + +# Update the TEST_CONFIG with the user supplied values. +TEST_CONFIG.update(TEST_CONFIG_OVERRIDE) + + +def get_pytest_env_vars() -> Dict[str, str]: + """Returns a dict for pytest invocation.""" + ret = {} + + # Override the GCLOUD_PROJECT and the alias. + env_key = TEST_CONFIG["gcloud_project_env"] + # This should error out if not set. + ret["GOOGLE_CLOUD_PROJECT"] = os.environ[env_key] + + # Apply user supplied envs. + ret.update(TEST_CONFIG["envs"]) + return ret + + +# DO NOT EDIT - automatically generated. +# All versions used to test samples. +ALL_VERSIONS = ["3.6", "3.7", "3.8", "3.9"] + +# Any default versions that should be ignored. +IGNORED_VERSIONS = TEST_CONFIG["ignored_versions"] + +TESTED_VERSIONS = sorted([v for v in ALL_VERSIONS if v not in IGNORED_VERSIONS]) + +INSTALL_LIBRARY_FROM_SOURCE = os.environ.get("INSTALL_LIBRARY_FROM_SOURCE", False) in ( + "True", + "true", +) +# +# Style Checks +# + + +def _determine_local_import_names(start_dir: str) -> List[str]: + """Determines all import names that should be considered "local". + + This is used when running the linter to insure that import order is + properly checked. + """ + file_ext_pairs = [os.path.splitext(path) for path in os.listdir(start_dir)] + return [ + basename + for basename, extension in file_ext_pairs + if extension == ".py" + or os.path.isdir(os.path.join(start_dir, basename)) + and basename not in ("__pycache__") + ] + + +# Linting with flake8. +# +# We ignore the following rules: +# E203: whitespace before ‘:’ +# E266: too many leading ‘#’ for block comment +# E501: line too long +# I202: Additional newline in a section of imports +# +# We also need to specify the rules which are ignored by default: +# ['E226', 'W504', 'E126', 'E123', 'W503', 'E24', 'E704', 'E121'] +FLAKE8_COMMON_ARGS = [ + "--show-source", + "--builtin=gettext", + "--max-complexity=20", + "--import-order-style=google", + "--exclude=.nox,.cache,env,lib,generated_pb2,*_pb2.py,*_pb2_grpc.py", + "--ignore=E121,E123,E126,E203,E226,E24,E266,E501,E704,W503,W504,I202", + "--max-line-length=88", +] + + +@nox.session +def lint(session: nox.sessions.Session) -> None: + if not TEST_CONFIG["enforce_type_hints"]: + session.install("flake8", "flake8-import-order") + else: + session.install("flake8", "flake8-import-order", "flake8-annotations") + + local_names = _determine_local_import_names(".") + args = FLAKE8_COMMON_ARGS + [ + "--application-import-names", + ",".join(local_names), + ".", + ] + session.run("flake8", *args) + + +# +# Black +# + + +@nox.session +def blacken(session: nox.sessions.Session) -> None: + session.install(BLACK_VERSION) + python_files = [path for path in os.listdir(".") if path.endswith(".py")] + + session.run("black", *python_files) + + +# +# Sample Tests +# + + +PYTEST_COMMON_ARGS = ["--junitxml=sponge_log.xml"] + + +def _session_tests( + session: nox.sessions.Session, post_install: Callable = None +) -> None: + if TEST_CONFIG["pip_version_override"]: + pip_version = TEST_CONFIG["pip_version_override"] + session.install(f"pip=={pip_version}") + """Runs py.test for a particular project.""" + if os.path.exists("requirements.txt"): + if os.path.exists("constraints.txt"): + session.install("-r", "requirements.txt", "-c", "constraints.txt") + else: + session.install("-r", "requirements.txt") + + if os.path.exists("requirements-test.txt"): + if os.path.exists("constraints-test.txt"): + session.install("-r", "requirements-test.txt", "-c", "constraints-test.txt") + else: + session.install("-r", "requirements-test.txt") + + if INSTALL_LIBRARY_FROM_SOURCE: + session.install("-e", _get_repo_root()) + + if post_install: + post_install(session) + + session.run( + "pytest", + *(PYTEST_COMMON_ARGS + session.posargs), + # Pytest will return 5 when no tests are collected. This can happen + # on travis where slow and flaky tests are excluded. + # See http://doc.pytest.org/en/latest/_modules/_pytest/main.html + success_codes=[0, 5], + env=get_pytest_env_vars(), + ) + + +@nox.session(python=ALL_VERSIONS) +def py(session: nox.sessions.Session) -> None: + """Runs py.test for a sample using the specified version of Python.""" + if session.python in TESTED_VERSIONS: + _session_tests(session) + else: + session.skip( + "SKIPPED: {} tests are disabled for this sample.".format(session.python) + ) + + +# +# Readmegen +# + + +def _get_repo_root() -> Optional[str]: + """ Returns the root folder of the project. """ + # Get root of this repository. Assume we don't have directories nested deeper than 10 items. + p = Path(os.getcwd()) + for i in range(10): + if p is None: + break + if Path(p / ".git").exists(): + return str(p) + # .git is not available in repos cloned via Cloud Build + # setup.py is always in the library's root, so use that instead + # https://github.com/googleapis/synthtool/issues/792 + if Path(p / "setup.py").exists(): + return str(p) + p = p.parent + raise Exception("Unable to detect repository root.") + + +GENERATED_READMES = sorted([x for x in Path(".").rglob("*.rst.in")]) + + +@nox.session +@nox.parametrize("path", GENERATED_READMES) +def readmegen(session: nox.sessions.Session, path: str) -> None: + """(Re-)generates the readme for a sample.""" + session.install("jinja2", "pyyaml") + dir_ = os.path.dirname(path) + + if os.path.exists(os.path.join(dir_, "requirements.txt")): + session.install("-r", os.path.join(dir_, "requirements.txt")) + + in_file = os.path.join(dir_, "README.rst.in") + session.run( + "python", _get_repo_root() + "/scripts/readme-gen/readme_gen.py", in_file + ) diff --git a/samples/magics/query.py b/samples/magics/query.py new file mode 100644 index 000000000..c2739eace --- /dev/null +++ b/samples/magics/query.py @@ -0,0 +1,37 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import IPython + +from . import _helpers + + +def query(): + ip = IPython.get_ipython() + ip.extension_manager.load_extension("google.cloud.bigquery") + + sample = """ + # [START bigquery_jupyter_query] + %%bigquery + SELECT name, SUM(number) as count + FROM `bigquery-public-data.usa_names.usa_1910_current` + GROUP BY name + ORDER BY count DESC + LIMIT 3 + # [END bigquery_jupyter_query] + """ + result = ip.run_cell(_helpers.strip_region_tags(sample)) + result.raise_error() # Throws an exception if the cell failed. + df = ip.user_ns["_"] # Retrieves last returned object in notebook session + return df diff --git a/samples/magics/query_params_scalars.py b/samples/magics/query_params_scalars.py new file mode 100644 index 000000000..a26f25aea --- /dev/null +++ b/samples/magics/query_params_scalars.py @@ -0,0 +1,38 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import IPython + +from . import _helpers + + +def query_with_parameters(): + ip = IPython.get_ipython() + ip.extension_manager.load_extension("google.cloud.bigquery") + + sample = """ + # [START bigquery_jupyter_query_params_scalars] + %%bigquery --params {"corpus_name": "hamlet", "limit": 10} + SELECT word, SUM(word_count) as count + FROM `bigquery-public-data.samples.shakespeare` + WHERE corpus = @corpus_name + GROUP BY word + ORDER BY count DESC + LIMIT @limit + # [END bigquery_jupyter_query_params_scalars] + """ + result = ip.run_cell(_helpers.strip_region_tags(sample)) + result.raise_error() # Throws an exception if the cell failed. + df = ip.user_ns["_"] # Retrieves last returned object in notebook session + return df diff --git a/samples/magics/query_params_scalars_test.py b/samples/magics/query_params_scalars_test.py new file mode 100644 index 000000000..9b4159667 --- /dev/null +++ b/samples/magics/query_params_scalars_test.py @@ -0,0 +1,23 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pandas + +from . import query_params_scalars + + +def test_query_with_parameters(): + df = query_params_scalars.query_with_parameters() + assert isinstance(df, pandas.DataFrame) + assert len(df) == 10 diff --git a/samples/magics/query_test.py b/samples/magics/query_test.py new file mode 100644 index 000000000..d20797908 --- /dev/null +++ b/samples/magics/query_test.py @@ -0,0 +1,23 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pandas + +from . import query + + +def test_query(): + df = query.query() + assert isinstance(df, pandas.DataFrame) + assert len(df) == 3 diff --git a/samples/magics/requirements-test.txt b/samples/magics/requirements-test.txt new file mode 100644 index 000000000..caa48813a --- /dev/null +++ b/samples/magics/requirements-test.txt @@ -0,0 +1,3 @@ +google-cloud-testutils==1.1.0 +pytest==6.2.5 +mock==4.0.3 diff --git a/samples/magics/requirements.txt b/samples/magics/requirements.txt new file mode 100644 index 000000000..f9b9d023c --- /dev/null +++ b/samples/magics/requirements.txt @@ -0,0 +1,12 @@ +google-cloud-bigquery==2.27.1 +google-cloud-bigquery-storage==2.9.0 +google-auth-oauthlib==0.4.6 +grpcio==1.41.0 +ipython==7.16.1; python_version < '3.7' +ipython==7.17.0; python_version >= '3.7' +matplotlib==3.3.4; python_version < '3.7' +matplotlib==3.4.1; python_version >= '3.7' +pandas==1.1.5; python_version < '3.7' +pandas==1.3.2; python_version >= '3.7' +pyarrow==5.0.0 +pytz==2021.1 From 4ecd64b04a08a5da79f8d981a5e232cd9ad0a773 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Mon, 1 Nov 2021 09:17:36 -0500 Subject: [PATCH 3/6] chore: use gapic-generator-python 0.53.4 (#1037) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore: use gapic-generator-python 0.53.4 docs: list oneofs in docstring committer: busunkim96@ PiperOrigin-RevId: 406468269 Source-Link: https://github.com/googleapis/googleapis/commit/83d81b0c8fc22291a13398d6d77f02dc97a5b6f4 Source-Link: https://github.com/googleapis/googleapis-gen/commit/2ff001fbacb9e77e71d734de5f955c05fdae8526 Copy-Tag: eyJwIjoiLmdpdGh1Yi8uT3dsQm90LnlhbWwiLCJoIjoiMmZmMDAxZmJhY2I5ZTc3ZTcxZDczNGRlNWY5NTVjMDVmZGFlODUyNiJ9 * 🦉 Updates from OwlBot See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md Co-authored-by: Owl Bot --- google/cloud/bigquery_v2/types/model.py | 22 +++++++++++++++++++ .../cloud/bigquery_v2/types/standard_sql.py | 9 ++++++++ samples/magics/noxfile.py | 6 ++++- 3 files changed, 36 insertions(+), 1 deletion(-) diff --git a/google/cloud/bigquery_v2/types/model.py b/google/cloud/bigquery_v2/types/model.py index 6e3ca0095..a56b21491 100644 --- a/google/cloud/bigquery_v2/types/model.py +++ b/google/cloud/bigquery_v2/types/model.py @@ -560,14 +560,23 @@ class Cluster(proto.Message): class FeatureValue(proto.Message): r"""Representative value of a single feature within the cluster. + This message has `oneof`_ fields (mutually exclusive fields). + For each oneof, at most one member field can be set at the same time. + Setting any member of the oneof automatically clears all other + members. + + .. _oneof: https://proto-plus-python.readthedocs.io/en/stable/fields.html#oneofs-mutually-exclusive-fields + Attributes: feature_column (str): The feature column name. numerical_value (google.protobuf.wrappers_pb2.DoubleValue): The numerical feature value. This is the centroid value for this feature. + This field is a member of `oneof`_ ``value``. categorical_value (google.cloud.bigquery_v2.types.Model.ClusteringMetrics.Cluster.FeatureValue.CategoricalValue): The categorical feature value. + This field is a member of `oneof`_ ``value``. """ class CategoricalValue(proto.Message): @@ -784,23 +793,36 @@ class EvaluationMetrics(proto.Message): data was used during training. These are not present for imported models. + This message has `oneof`_ fields (mutually exclusive fields). + For each oneof, at most one member field can be set at the same time. + Setting any member of the oneof automatically clears all other + members. + + .. _oneof: https://proto-plus-python.readthedocs.io/en/stable/fields.html#oneofs-mutually-exclusive-fields + Attributes: regression_metrics (google.cloud.bigquery_v2.types.Model.RegressionMetrics): Populated for regression models and explicit feedback type matrix factorization models. + This field is a member of `oneof`_ ``metrics``. binary_classification_metrics (google.cloud.bigquery_v2.types.Model.BinaryClassificationMetrics): Populated for binary classification/classifier models. + This field is a member of `oneof`_ ``metrics``. multi_class_classification_metrics (google.cloud.bigquery_v2.types.Model.MultiClassClassificationMetrics): Populated for multi-class classification/classifier models. + This field is a member of `oneof`_ ``metrics``. clustering_metrics (google.cloud.bigquery_v2.types.Model.ClusteringMetrics): Populated for clustering models. + This field is a member of `oneof`_ ``metrics``. ranking_metrics (google.cloud.bigquery_v2.types.Model.RankingMetrics): Populated for implicit feedback type matrix factorization models. + This field is a member of `oneof`_ ``metrics``. arima_forecasting_metrics (google.cloud.bigquery_v2.types.Model.ArimaForecastingMetrics): Populated for ARIMA models. + This field is a member of `oneof`_ ``metrics``. """ regression_metrics = proto.Field( diff --git a/google/cloud/bigquery_v2/types/standard_sql.py b/google/cloud/bigquery_v2/types/standard_sql.py index 69a221c3c..d6c133634 100644 --- a/google/cloud/bigquery_v2/types/standard_sql.py +++ b/google/cloud/bigquery_v2/types/standard_sql.py @@ -35,6 +35,13 @@ class StandardSqlDataType(proto.Message): type={type_kind="STRING"}}, {name="y", type={type_kind="ARRAY", array_element_type="DATE"}} ]}} + This message has `oneof`_ fields (mutually exclusive fields). + For each oneof, at most one member field can be set at the same time. + Setting any member of the oneof automatically clears all other + members. + + .. _oneof: https://proto-plus-python.readthedocs.io/en/stable/fields.html#oneofs-mutually-exclusive-fields + Attributes: type_kind (google.cloud.bigquery_v2.types.StandardSqlDataType.TypeKind): Required. The top level type of this field. @@ -42,9 +49,11 @@ class StandardSqlDataType(proto.Message): "INT64", "DATE", "ARRAY"). array_element_type (google.cloud.bigquery_v2.types.StandardSqlDataType): The type of the array's elements, if type_kind = "ARRAY". + This field is a member of `oneof`_ ``sub_type``. struct_type (google.cloud.bigquery_v2.types.StandardSqlStructType): The fields of this struct, in order, if type_kind = "STRUCT". + This field is a member of `oneof`_ ``sub_type``. """ class TypeKind(proto.Enum): diff --git a/samples/magics/noxfile.py b/samples/magics/noxfile.py index b008613f0..93a9122cc 100644 --- a/samples/magics/noxfile.py +++ b/samples/magics/noxfile.py @@ -87,7 +87,7 @@ def get_pytest_env_vars() -> Dict[str, str]: # DO NOT EDIT - automatically generated. # All versions used to test samples. -ALL_VERSIONS = ["3.6", "3.7", "3.8", "3.9"] +ALL_VERSIONS = ["3.6", "3.7", "3.8", "3.9", "3.10"] # Any default versions that should be ignored. IGNORED_VERSIONS = TEST_CONFIG["ignored_versions"] @@ -98,6 +98,10 @@ def get_pytest_env_vars() -> Dict[str, str]: "True", "true", ) + +# Error if a python version is missing +nox.options.error_on_missing_interpreters = True + # # Style Checks # From fe16adc86a170d0992c32091b349b036f8b43884 Mon Sep 17 00:00:00 2001 From: Steffany Brown <30247553+steffnay@users.noreply.github.com> Date: Mon, 1 Nov 2021 08:48:12 -0700 Subject: [PATCH 4/6] feat: accept TableListItem where TableReference is accepted (#1016) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Thank you for opening a Pull Request! Before submitting your PR, there are a few things you can do to make sure it goes smoothly: - [ ] Make sure to open an issue as a [bug/issue](https://github.com/googleapis/python-bigquery/issues/new/choose) before writing your code! That way we can discuss the change, evaluate designs, and agree on the general idea - [ ] Ensure the tests and linter pass - [ ] Code coverage does not decrease (if any source code was changed) - [ ] Appropriate docs were updated (if necessary) 🦕 --- google/cloud/bigquery/client.py | 50 ++++++++++++++++++++------------- tests/unit/test_client.py | 6 ++-- 2 files changed, 34 insertions(+), 22 deletions(-) diff --git a/google/cloud/bigquery/client.py b/google/cloud/bigquery/client.py index 9cb6af8f0..4bdd43e8f 100644 --- a/google/cloud/bigquery/client.py +++ b/google/cloud/bigquery/client.py @@ -806,13 +806,12 @@ def get_dataset( def get_iam_policy( self, - table: Union[Table, TableReference], + table: Union[Table, TableReference, TableListItem, str], requested_policy_version: int = 1, retry: retries.Retry = DEFAULT_RETRY, timeout: float = DEFAULT_TIMEOUT, ) -> Policy: - if not isinstance(table, (Table, TableReference)): - raise TypeError("table must be a Table or TableReference") + table = _table_arg_to_table_ref(table, default_project=self.project) if requested_policy_version != 1: raise ValueError("only IAM policy version 1 is supported") @@ -835,14 +834,13 @@ def get_iam_policy( def set_iam_policy( self, - table: Union[Table, TableReference], + table: Union[Table, TableReference, TableListItem, str], policy: Policy, updateMask: str = None, retry: retries.Retry = DEFAULT_RETRY, timeout: float = DEFAULT_TIMEOUT, ) -> Policy: - if not isinstance(table, (Table, TableReference)): - raise TypeError("table must be a Table or TableReference") + table = _table_arg_to_table_ref(table, default_project=self.project) if not isinstance(policy, (Policy)): raise TypeError("policy must be a Policy") @@ -869,13 +867,12 @@ def set_iam_policy( def test_iam_permissions( self, - table: Union[Table, TableReference], + table: Union[Table, TableReference, TableListItem, str], permissions: Sequence[str], retry: retries.Retry = DEFAULT_RETRY, timeout: float = DEFAULT_TIMEOUT, ) -> Dict[str, Any]: - if not isinstance(table, (Table, TableReference)): - raise TypeError("table must be a Table or TableReference") + table = _table_arg_to_table_ref(table, default_project=self.project) body = {"permissions": permissions} @@ -982,7 +979,7 @@ def get_routine( def get_table( self, - table: Union[Table, TableReference, str], + table: Union[Table, TableReference, TableListItem, str], retry: retries.Retry = DEFAULT_RETRY, timeout: float = DEFAULT_TIMEOUT, ) -> Table: @@ -992,6 +989,7 @@ def get_table( table (Union[ \ google.cloud.bigquery.table.Table, \ google.cloud.bigquery.table.TableReference, \ + google.cloud.bigquery.table.TableListItem, \ str, \ ]): A reference to the table to fetch from the BigQuery API. @@ -1757,7 +1755,7 @@ def delete_routine( def delete_table( self, - table: Union[Table, TableReference, str], + table: Union[Table, TableReference, TableListItem, str], retry: retries.Retry = DEFAULT_RETRY, timeout: float = DEFAULT_TIMEOUT, not_found_ok: bool = False, @@ -1771,6 +1769,7 @@ def delete_table( table (Union[ \ google.cloud.bigquery.table.Table, \ google.cloud.bigquery.table.TableReference, \ + google.cloud.bigquery.table.TableListItem, \ str, \ ]): A reference to the table to delete. If a string is passed in, @@ -2257,7 +2256,7 @@ def api_request(*args, **kwargs): def load_table_from_uri( self, source_uris: Union[str, Sequence[str]], - destination: Union[Table, TableReference, str], + destination: Union[Table, TableReference, TableListItem, str], job_id: str = None, job_id_prefix: str = None, location: str = None, @@ -2278,6 +2277,7 @@ def load_table_from_uri( destination (Union[ \ google.cloud.bigquery.table.Table, \ google.cloud.bigquery.table.TableReference, \ + google.cloud.bigquery.table.TableListItem, \ str, \ ]): Table into which data is to be loaded. If a string is passed @@ -2339,7 +2339,7 @@ def load_table_from_uri( def load_table_from_file( self, file_obj: BinaryIO, - destination: Union[Table, TableReference, str], + destination: Union[Table, TableReference, TableListItem, str], rewind: bool = False, size: int = None, num_retries: int = _DEFAULT_NUM_RETRIES, @@ -2360,6 +2360,7 @@ def load_table_from_file( destination (Union[ \ google.cloud.bigquery.table.Table, \ google.cloud.bigquery.table.TableReference, \ + google.cloud.bigquery.table.TableListItem, \ str, \ ]): Table into which data is to be loaded. If a string is passed @@ -2699,7 +2700,7 @@ def load_table_from_dataframe( def load_table_from_json( self, json_rows: Iterable[Dict[str, Any]], - destination: Union[Table, TableReference, str], + destination: Union[Table, TableReference, TableListItem, str], num_retries: int = _DEFAULT_NUM_RETRIES, job_id: str = None, job_id_prefix: str = None, @@ -2733,6 +2734,7 @@ def load_table_from_json( destination (Union[ \ google.cloud.bigquery.table.Table, \ google.cloud.bigquery.table.TableReference, \ + google.cloud.bigquery.table.TableListItem, \ str, \ ]): Table into which data is to be loaded. If a string is passed @@ -2980,9 +2982,13 @@ def _do_multipart_upload( def copy_table( self, sources: Union[ - Table, TableReference, str, Sequence[Union[Table, TableReference, str]] + Table, + TableReference, + TableListItem, + str, + Sequence[Union[Table, TableReference, TableListItem, str]], ], - destination: Union[Table, TableReference, str], + destination: Union[Table, TableReference, TableListItem, str], job_id: str = None, job_id_prefix: str = None, location: str = None, @@ -3000,11 +3006,13 @@ def copy_table( sources (Union[ \ google.cloud.bigquery.table.Table, \ google.cloud.bigquery.table.TableReference, \ + google.cloud.bigquery.table.TableListItem, \ str, \ Sequence[ \ Union[ \ google.cloud.bigquery.table.Table, \ google.cloud.bigquery.table.TableReference, \ + google.cloud.bigquery.table.TableListItem, \ str, \ ] \ ], \ @@ -3013,6 +3021,7 @@ def copy_table( destination (Union[ \ google.cloud.bigquery.table.Table, \ google.cloud.bigquery.table.TableReference, \ + google.cloud.bigquery.table.TableListItem, \ str, \ ]): Table into which data is to be copied. @@ -3084,7 +3093,7 @@ def copy_table( def extract_table( self, - source: Union[Table, TableReference, Model, ModelReference, str], + source: Union[Table, TableReference, TableListItem, Model, ModelReference, str], destination_uris: Union[str, Sequence[str]], job_id: str = None, job_id_prefix: str = None, @@ -3104,6 +3113,7 @@ def extract_table( source (Union[ \ google.cloud.bigquery.table.Table, \ google.cloud.bigquery.table.TableReference, \ + google.cloud.bigquery.table.TableListItem, \ google.cloud.bigquery.model.Model, \ google.cloud.bigquery.model.ModelReference, \ src, \ @@ -3465,7 +3475,7 @@ def insert_rows_from_dataframe( def insert_rows_json( self, - table: Union[Table, TableReference, str], + table: Union[Table, TableReference, TableListItem, str], json_rows: Sequence[Dict], row_ids: Union[Iterable[str], AutoRowIDs, None] = AutoRowIDs.GENERATE_UUID, skip_invalid_rows: bool = None, @@ -3483,6 +3493,7 @@ def insert_rows_json( table (Union[ \ google.cloud.bigquery.table.Table \ google.cloud.bigquery.table.TableReference, \ + google.cloud.bigquery.table.TableListItem, \ str \ ]): The destination table for the row data, or a reference to it. @@ -3605,7 +3616,7 @@ def insert_rows_json( def list_partitions( self, - table: Union[Table, TableReference, str], + table: Union[Table, TableReference, TableListItem, str], retry: retries.Retry = DEFAULT_RETRY, timeout: float = DEFAULT_TIMEOUT, ) -> Sequence[str]: @@ -3615,6 +3626,7 @@ def list_partitions( table (Union[ \ google.cloud.bigquery.table.Table, \ google.cloud.bigquery.table.TableReference, \ + google.cloud.bigquery.table.TableListItem, \ str, \ ]): The table or reference from which to get partition info diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index 48dacf7e2..11b336728 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -1554,7 +1554,7 @@ def test_get_iam_policy_w_invalid_table(self): self.PROJECT, self.DS_ID, self.TABLE_ID, ) - with self.assertRaises(TypeError): + with self.assertRaises(ValueError): client.get_iam_policy(table_resource_string) def test_get_iam_policy_w_invalid_version(self): @@ -1675,7 +1675,7 @@ def test_set_iam_policy_w_invalid_table(self): self.TABLE_ID, ) - with self.assertRaises(TypeError): + with self.assertRaises(ValueError): client.set_iam_policy(table_resource_string, policy) def test_test_iam_permissions(self): @@ -1717,7 +1717,7 @@ def test_test_iam_permissions_w_invalid_table(self): PERMISSIONS = ["bigquery.tables.get", "bigquery.tables.update"] - with self.assertRaises(TypeError): + with self.assertRaises(ValueError): client.test_iam_permissions(table_resource_string, PERMISSIONS) def test_update_dataset_w_invalid_field(self): From 5bbb832a83ebb66db4b5ee740cdfc53f4df8430b Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Wed, 3 Nov 2021 14:09:53 -0500 Subject: [PATCH 5/6] feat: support Python 3.10 (#1043) * feat: support Python 3.10 * fix pandas deps to match reality * run system tests with 3.10 * avoid geopandas on Python 3.10 * install google-cloud-bigquery from source * update kokoro configs * remove Python 2.7 config --- .kokoro/presubmit/prerelease-deps-3.8.cfg | 4 ++-- .kokoro/presubmit/snippets-3.10.cfg | 7 +++++++ .../{snippets-2.7.cfg => system-3.10.cfg} | 2 +- noxfile.py | 17 +++++++++++++---- samples/geography/noxfile_config.py | 7 ++++++- samples/geography/requirements.txt | 8 +++----- samples/magics/requirements.txt | 9 ++++----- samples/snippets/requirements.txt | 7 +++---- setup.py | 5 +++-- 9 files changed, 42 insertions(+), 24 deletions(-) create mode 100644 .kokoro/presubmit/snippets-3.10.cfg rename .kokoro/presubmit/{snippets-2.7.cfg => system-3.10.cfg} (82%) diff --git a/.kokoro/presubmit/prerelease-deps-3.8.cfg b/.kokoro/presubmit/prerelease-deps-3.8.cfg index f06806baf..fabe3e347 100644 --- a/.kokoro/presubmit/prerelease-deps-3.8.cfg +++ b/.kokoro/presubmit/prerelease-deps-3.8.cfg @@ -3,5 +3,5 @@ # Only run this nox session. env_vars: { key: "NOX_SESSION" - value: "prerelease_deps" -} \ No newline at end of file + value: "prerelease_deps-3.8" +} diff --git a/.kokoro/presubmit/snippets-3.10.cfg b/.kokoro/presubmit/snippets-3.10.cfg new file mode 100644 index 000000000..dde182fb9 --- /dev/null +++ b/.kokoro/presubmit/snippets-3.10.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Only run this nox session. +env_vars: { + key: "NOX_SESSION" + value: "snippets-3.10" +} diff --git a/.kokoro/presubmit/snippets-2.7.cfg b/.kokoro/presubmit/system-3.10.cfg similarity index 82% rename from .kokoro/presubmit/snippets-2.7.cfg rename to .kokoro/presubmit/system-3.10.cfg index 3bd6134d2..30956a3ab 100644 --- a/.kokoro/presubmit/snippets-2.7.cfg +++ b/.kokoro/presubmit/system-3.10.cfg @@ -3,5 +3,5 @@ # Only run this nox session. env_vars: { key: "NOX_SESSION" - value: "snippets-2.7" + value: "system-3.10" } diff --git a/noxfile.py b/noxfile.py index 64eacaff5..6f04940c9 100644 --- a/noxfile.py +++ b/noxfile.py @@ -27,8 +27,8 @@ BLACK_PATHS = ("docs", "google", "samples", "tests", "noxfile.py", "setup.py") DEFAULT_PYTHON_VERSION = "3.8" -SYSTEM_TEST_PYTHON_VERSIONS = ["3.8"] -UNIT_TEST_PYTHON_VERSIONS = ["3.6", "3.7", "3.8", "3.9"] +SYSTEM_TEST_PYTHON_VERSIONS = ["3.8", "3.10"] +UNIT_TEST_PYTHON_VERSIONS = ["3.6", "3.7", "3.8", "3.9", "3.10"] CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute() # 'docfx' is excluded since it only needs to run in 'docs-presubmit' @@ -69,7 +69,12 @@ def default(session, install_extras=True): constraints_path, ) - install_target = ".[all]" if install_extras else "." + if install_extras and session.python == "3.10": + install_target = ".[bqstorage,pandas,tqdm,opentelemetry]" + elif install_extras: + install_target = ".[all]" + else: + install_target = "." session.install("-e", install_target, "-c", constraints_path) session.install("ipython", "-c", constraints_path) @@ -153,7 +158,11 @@ def system(session): # Data Catalog needed for the column ACL test with a real Policy Tag. session.install("google-cloud-datacatalog", "-c", constraints_path) - session.install("-e", ".[all]", "-c", constraints_path) + if session.python == "3.10": + extras = "[bqstorage,pandas,tqdm,opentelemetry]" + else: + extras = "[all]" + session.install("-e", f".{extras}", "-c", constraints_path) session.install("ipython", "-c", constraints_path) # Run py.test against the system tests. diff --git a/samples/geography/noxfile_config.py b/samples/geography/noxfile_config.py index 7d2e02346..315bd5be8 100644 --- a/samples/geography/noxfile_config.py +++ b/samples/geography/noxfile_config.py @@ -22,7 +22,12 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7"], + "ignored_versions": [ + "2.7", + # TODO: Enable 3.10 once there is a geopandas/fiona release. + # https://github.com/Toblerity/Fiona/issues/1043 + "3.10", + ], # An envvar key for determining the project id to use. Change it # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a # build specific Cloud project. You can also use your own string diff --git a/samples/geography/requirements.txt b/samples/geography/requirements.txt index ecd428ab9..e2de86673 100644 --- a/samples/geography/requirements.txt +++ b/samples/geography/requirements.txt @@ -24,14 +24,12 @@ importlib-metadata==4.8.1 libcst==0.3.21 munch==2.5.0 mypy-extensions==0.4.3 -numpy==1.19.5; python_version < "3.7" -numpy==1.21.2; python_version > "3.6" packaging==21.0 pandas==1.1.5; python_version < '3.7' -pandas==1.3.2; python_version >= '3.7' +pandas==1.3.4; python_version >= '3.7' proto-plus==1.19.2 protobuf==3.18.0 -pyarrow==5.0.0 +pyarrow==6.0.0 pyasn1==0.4.8 pyasn1-modules==0.2.8 pycparser==2.20 @@ -43,7 +41,7 @@ pytz==2021.1 PyYAML==5.4.1 requests==2.26.0 rsa==4.7.2 -Shapely==1.7.1 +Shapely==1.8.0 six==1.16.0 typing-extensions==3.10.0.2 typing-inspect==0.7.1 diff --git a/samples/magics/requirements.txt b/samples/magics/requirements.txt index f9b9d023c..5cc7ec33f 100644 --- a/samples/magics/requirements.txt +++ b/samples/magics/requirements.txt @@ -1,12 +1,11 @@ -google-cloud-bigquery==2.27.1 google-cloud-bigquery-storage==2.9.0 google-auth-oauthlib==0.4.6 grpcio==1.41.0 ipython==7.16.1; python_version < '3.7' -ipython==7.17.0; python_version >= '3.7' +ipython==7.29.0; python_version >= '3.7' matplotlib==3.3.4; python_version < '3.7' -matplotlib==3.4.1; python_version >= '3.7' +matplotlib==3.5.0rc1; python_version >= '3.7' pandas==1.1.5; python_version < '3.7' -pandas==1.3.2; python_version >= '3.7' -pyarrow==5.0.0 +pandas==1.3.4; python_version >= '3.7' +pyarrow==6.0.0 pytz==2021.1 diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index f9b9d023c..f79552392 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,12 +1,11 @@ -google-cloud-bigquery==2.27.1 google-cloud-bigquery-storage==2.9.0 google-auth-oauthlib==0.4.6 grpcio==1.41.0 ipython==7.16.1; python_version < '3.7' -ipython==7.17.0; python_version >= '3.7' +ipython==7.29.0; python_version >= '3.7' matplotlib==3.3.4; python_version < '3.7' matplotlib==3.4.1; python_version >= '3.7' pandas==1.1.5; python_version < '3.7' -pandas==1.3.2; python_version >= '3.7' -pyarrow==5.0.0 +pandas==1.3.4; python_version >= '3.7' +pyarrow==6.0.0 pytz==2021.1 diff --git a/setup.py b/setup.py index 95dad190a..db69c45b1 100644 --- a/setup.py +++ b/setup.py @@ -59,7 +59,7 @@ ] + pyarrow_dep, "geopandas": ["geopandas>=0.9.0, <1.0dev", "Shapely>=1.6.0, <2.0dev"], - "pandas": ["pandas>=0.23.0"] + pyarrow_dep, + "pandas": ["pandas>=0.24.2"] + pyarrow_dep, "bignumeric_type": pyarrow_dep, "tqdm": ["tqdm >= 4.7.4, <5.0.0dev"], "opentelemetry": [ @@ -127,6 +127,7 @@ "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", "Operating System :: OS Independent", "Topic :: Internet", ], @@ -135,7 +136,7 @@ namespace_packages=namespaces, install_requires=dependencies, extras_require=extras, - python_requires=">=3.6, <3.10", + python_requires=">=3.6, <3.11", include_package_data=True, zip_safe=False, ) From 7c5fa8a4a84a0462dc66e0ccdb00cb0da4aa1452 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Wed, 3 Nov 2021 19:34:15 +0000 Subject: [PATCH 6/6] chore: release 2.30.0 (#1039) :robot: I have created a release \*beep\* \*boop\* --- ## [2.30.0](https://www.github.com/googleapis/python-bigquery/compare/v2.29.0...v2.30.0) (2021-11-03) ### Features * accept TableListItem where TableReference is accepted ([#1016](https://www.github.com/googleapis/python-bigquery/issues/1016)) ([fe16adc](https://www.github.com/googleapis/python-bigquery/commit/fe16adc86a170d0992c32091b349b036f8b43884)) * support Python 3.10 ([#1043](https://www.github.com/googleapis/python-bigquery/issues/1043)) ([5bbb832](https://www.github.com/googleapis/python-bigquery/commit/5bbb832a83ebb66db4b5ee740cdfc53f4df8430b)) ### Documentation * add code samples for Jupyter/IPython magics ([#1013](https://www.github.com/googleapis/python-bigquery/issues/1013)) ([61141ee](https://www.github.com/googleapis/python-bigquery/commit/61141ee0634024ad261d1595c95cd14a896fb87e)) * **samples:** add create external table with hive partitioning ([#1033](https://www.github.com/googleapis/python-bigquery/issues/1033)) ([d64f5b6](https://www.github.com/googleapis/python-bigquery/commit/d64f5b682854a2293244426316890df4ab1e079e)) --- This PR was generated with [Release Please](https://github.com/googleapis/release-please). See [documentation](https://github.com/googleapis/release-please#release-please). --- CHANGELOG.md | 14 ++++++++++++++ google/cloud/bigquery/version.py | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d45d501d..b3bbee86a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,20 @@ [1]: https://pypi.org/project/google-cloud-bigquery/#history +## [2.30.0](https://www.github.com/googleapis/python-bigquery/compare/v2.29.0...v2.30.0) (2021-11-03) + + +### Features + +* accept TableListItem where TableReference is accepted ([#1016](https://www.github.com/googleapis/python-bigquery/issues/1016)) ([fe16adc](https://www.github.com/googleapis/python-bigquery/commit/fe16adc86a170d0992c32091b349b036f8b43884)) +* support Python 3.10 ([#1043](https://www.github.com/googleapis/python-bigquery/issues/1043)) ([5bbb832](https://www.github.com/googleapis/python-bigquery/commit/5bbb832a83ebb66db4b5ee740cdfc53f4df8430b)) + + +### Documentation + +* add code samples for Jupyter/IPython magics ([#1013](https://www.github.com/googleapis/python-bigquery/issues/1013)) ([61141ee](https://www.github.com/googleapis/python-bigquery/commit/61141ee0634024ad261d1595c95cd14a896fb87e)) +* **samples:** add create external table with hive partitioning ([#1033](https://www.github.com/googleapis/python-bigquery/issues/1033)) ([d64f5b6](https://www.github.com/googleapis/python-bigquery/commit/d64f5b682854a2293244426316890df4ab1e079e)) + ## [2.29.0](https://www.github.com/googleapis/python-bigquery/compare/v2.28.1...v2.29.0) (2021-10-27) diff --git a/google/cloud/bigquery/version.py b/google/cloud/bigquery/version.py index c8ba30be0..e89661993 100644 --- a/google/cloud/bigquery/version.py +++ b/google/cloud/bigquery/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.29.0" +__version__ = "2.30.0"