diff --git a/.github/sync-repo-settings.yaml b/.github/sync-repo-settings.yaml
index 8634a3043..6572e5982 100644
--- a/.github/sync-repo-settings.yaml
+++ b/.github/sync-repo-settings.yaml
@@ -1,9 +1,9 @@
-# https://github.com/googleapis/repo-automation-bots/tree/master/packages/sync-repo-settings
-# Rules for master branch protection
+# https://github.com/googleapis/repo-automation-bots/tree/main/packages/sync-repo-settings
+# Rules for main branch protection
branchProtectionRules:
# Identifies the protection rule pattern. Name of the branch to be protected.
-# Defaults to `master`
-- pattern: master
+# Defaults to `main`
+- pattern: main
requiresCodeOwnerReviews: true
requiresStrictStatusChecks: true
requiredStatusCheckContexts:
diff --git a/.kokoro/build.sh b/.kokoro/build.sh
index 302cc1e1a..4d6a1d0f6 100755
--- a/.kokoro/build.sh
+++ b/.kokoro/build.sh
@@ -41,7 +41,7 @@ python3 -m pip install --upgrade --quiet nox
python3 -m nox --version
# If this is a continuous build, send the test log to the FlakyBot.
-# See https://github.com/googleapis/repo-automation-bots/tree/master/packages/flakybot.
+# See https://github.com/googleapis/repo-automation-bots/tree/main/packages/flakybot.
if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"continuous"* ]]; then
cleanup() {
chmod +x $KOKORO_GFILE_DIR/linux_amd64/flakybot
diff --git a/.kokoro/test-samples-impl.sh b/.kokoro/test-samples-impl.sh
index 311a8d54b..8a324c9c7 100755
--- a/.kokoro/test-samples-impl.sh
+++ b/.kokoro/test-samples-impl.sh
@@ -80,7 +80,7 @@ for file in samples/**/requirements.txt; do
EXIT=$?
# If this is a periodic build, send the test log to the FlakyBot.
- # See https://github.com/googleapis/repo-automation-bots/tree/master/packages/flakybot.
+ # See https://github.com/googleapis/repo-automation-bots/tree/main/packages/flakybot.
if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"periodic"* ]]; then
chmod +x $KOKORO_GFILE_DIR/linux_amd64/flakybot
$KOKORO_GFILE_DIR/linux_amd64/flakybot
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8a21df6fe..b4c8e5fb7 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,19 @@
[1]: https://pypi.org/project/google-cloud-bigquery/#history
+### [2.25.2](https://www.github.com/googleapis/python-bigquery/compare/v2.25.1...v2.25.2) (2021-08-31)
+
+
+### Bug Fixes
+
+* error inserting DataFrame with REPEATED field ([#925](https://www.github.com/googleapis/python-bigquery/issues/925)) ([656d2fa](https://www.github.com/googleapis/python-bigquery/commit/656d2fa6f870573a21235c83463752a2d084caba))
+* underscores weren't allowed in struct field names when passing parameters to the DB API ([#930](https://www.github.com/googleapis/python-bigquery/issues/930)) ([fcb0bc6](https://www.github.com/googleapis/python-bigquery/commit/fcb0bc68c972c2c98bb8542f54e9228308177ecb))
+
+
+### Documentation
+
+* update docstring for bigquery_create_routine sample ([#883](https://www.github.com/googleapis/python-bigquery/issues/883)) ([#917](https://www.github.com/googleapis/python-bigquery/issues/917)) ([e2d12b7](https://www.github.com/googleapis/python-bigquery/commit/e2d12b795ef2dc51b0ee36f1b3000edb1e64ce05))
+
### [2.25.1](https://www.github.com/googleapis/python-bigquery/compare/v2.25.0...v2.25.1) (2021-08-25)
diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst
index 2faf5aed3..5b87973dd 100644
--- a/CONTRIBUTING.rst
+++ b/CONTRIBUTING.rst
@@ -50,9 +50,9 @@ You'll have to create a development environment using a Git checkout:
# Configure remotes such that you can pull changes from the googleapis/python-bigquery
# repository into your local repository.
$ git remote add upstream git@github.com:googleapis/python-bigquery.git
- # fetch and merge changes from upstream into master
+ # fetch and merge changes from upstream into main
$ git fetch upstream
- $ git merge upstream/master
+ $ git merge upstream/main
Now your local repo is set up such that you will push changes to your GitHub
repo, from which you can submit a pull request.
@@ -110,12 +110,12 @@ Coding Style
variables::
export GOOGLE_CLOUD_TESTING_REMOTE="upstream"
- export GOOGLE_CLOUD_TESTING_BRANCH="master"
+ export GOOGLE_CLOUD_TESTING_BRANCH="main"
By doing this, you are specifying the location of the most up-to-date
version of ``python-bigquery``. The the suggested remote name ``upstream``
should point to the official ``googleapis`` checkout and the
- the branch should be the main branch on that remote (``master``).
+ the branch should be the main branch on that remote (``main``).
- This repository contains configuration for the
`pre-commit `__ tool, which automates checking
@@ -209,7 +209,7 @@ The `description on PyPI`_ for the project comes directly from the
``README``. Due to the reStructuredText (``rst``) parser used by
PyPI, relative links which will work on GitHub (e.g. ``CONTRIBUTING.rst``
instead of
-``https://github.com/googleapis/python-bigquery/blob/master/CONTRIBUTING.rst``)
+``https://github.com/googleapis/python-bigquery/blob/main/CONTRIBUTING.rst``)
may cause problems creating links or rendering the description.
.. _description on PyPI: https://pypi.org/project/google-cloud-bigquery
@@ -234,7 +234,7 @@ We support:
Supported versions can be found in our ``noxfile.py`` `config`_.
-.. _config: https://github.com/googleapis/python-bigquery/blob/master/noxfile.py
+.. _config: https://github.com/googleapis/python-bigquery/blob/main/noxfile.py
We also explicitly decided to support Python 3 beginning with version 3.6.
diff --git a/docs/conf.py b/docs/conf.py
index 59a2d8fb3..07e5d8c30 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -76,8 +76,8 @@
# The encoding of source files.
# source_encoding = 'utf-8-sig'
-# The master toctree document.
-master_doc = "index"
+# The root toctree document.
+root_doc = "index"
# General information about the project.
project = "google-cloud-bigquery"
@@ -281,7 +281,7 @@
# author, documentclass [howto, manual, or own class]).
latex_documents = [
(
- master_doc,
+ root_doc,
"google-cloud-bigquery.tex",
"google-cloud-bigquery Documentation",
author,
@@ -316,7 +316,7 @@
# (source start file, name, description, authors, manual section).
man_pages = [
(
- master_doc,
+ root_doc,
"google-cloud-bigquery",
"google-cloud-bigquery Documentation",
[author],
@@ -335,7 +335,7 @@
# dir menu entry, description, category)
texinfo_documents = [
(
- master_doc,
+ root_doc,
"google-cloud-bigquery",
"google-cloud-bigquery Documentation",
author,
diff --git a/google/cloud/bigquery/_pandas_helpers.py b/google/cloud/bigquery/_pandas_helpers.py
index ab58b1729..29139ae09 100644
--- a/google/cloud/bigquery/_pandas_helpers.py
+++ b/google/cloud/bigquery/_pandas_helpers.py
@@ -844,7 +844,13 @@ def dataframe_to_json_generator(dataframe):
output = {}
for column, value in zip(dataframe.columns, row):
# Omit NaN values.
- if pandas.isna(value):
+ is_nan = pandas.isna(value)
+
+ # isna() can also return an array-like of bools, but the latter's boolean
+ # value is ambiguous, hence an extra check. An array-like value is *not*
+ # considered a NaN, however.
+ if isinstance(is_nan, bool) and is_nan:
continue
output[column] = value
+
yield output
diff --git a/google/cloud/bigquery/dbapi/_helpers.py b/google/cloud/bigquery/dbapi/_helpers.py
index 9c134b47c..72e711bcf 100644
--- a/google/cloud/bigquery/dbapi/_helpers.py
+++ b/google/cloud/bigquery/dbapi/_helpers.py
@@ -173,7 +173,7 @@ def _parse_type(
\s*
(ARRAY|STRUCT|RECORD) # Type
\s*
- <([A-Z0-9<> ,()]+)> # Subtype(s)
+ <([A-Z0-9_<> ,()]+)> # Subtype(s)
\s*$
""",
re.IGNORECASE | re.VERBOSE,
diff --git a/google/cloud/bigquery/dbapi/cursor.py b/google/cloud/bigquery/dbapi/cursor.py
index 587598d5f..b1239ff57 100644
--- a/google/cloud/bigquery/dbapi/cursor.py
+++ b/google/cloud/bigquery/dbapi/cursor.py
@@ -494,7 +494,7 @@ def _extract_types(
([^:)]*) # name
(?:: # ':' introduces type
( # start of type group
- [a-zA-Z0-9<>, ]+ # First part, no parens
+ [a-zA-Z0-9_<>, ]+ # First part, no parens
(?: # start sets of parens + non-paren text
\([0-9 ,]+\) # comma-separated groups of digits in parens
diff --git a/google/cloud/bigquery/job/copy_.py b/google/cloud/bigquery/job/copy_.py
index c6ee98944..f0dd3d668 100644
--- a/google/cloud/bigquery/job/copy_.py
+++ b/google/cloud/bigquery/job/copy_.py
@@ -240,7 +240,7 @@ def to_api_repr(self):
def from_api_repr(cls, resource, client):
"""Factory: construct a job given its API representation
- .. note:
+ .. note::
This method assumes that the project found in the resource matches
the client's project.
diff --git a/google/cloud/bigquery/job/extract.py b/google/cloud/bigquery/job/extract.py
index 3373bcdef..52aa036c9 100644
--- a/google/cloud/bigquery/job/extract.py
+++ b/google/cloud/bigquery/job/extract.py
@@ -244,7 +244,7 @@ def to_api_repr(self):
def from_api_repr(cls, resource: dict, client) -> "ExtractJob":
"""Factory: construct a job given its API representation
- .. note:
+ .. note::
This method assumes that the project found in the resource matches
the client's project.
diff --git a/google/cloud/bigquery/job/load.py b/google/cloud/bigquery/job/load.py
index aee055c1c..b12c3e621 100644
--- a/google/cloud/bigquery/job/load.py
+++ b/google/cloud/bigquery/job/load.py
@@ -800,7 +800,7 @@ def to_api_repr(self):
def from_api_repr(cls, resource: dict, client) -> "LoadJob":
"""Factory: construct a job given its API representation
- .. note:
+ .. note::
This method assumes that the project found in the resource matches
the client's project.
diff --git a/google/cloud/bigquery/version.py b/google/cloud/bigquery/version.py
index 21cbec9fe..e8672849f 100644
--- a/google/cloud/bigquery/version.py
+++ b/google/cloud/bigquery/version.py
@@ -12,4 +12,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-__version__ = "2.25.1"
+__version__ = "2.25.2"
diff --git a/owlbot.py b/owlbot.py
index 8664b658a..09aa8ca6f 100644
--- a/owlbot.py
+++ b/owlbot.py
@@ -169,4 +169,52 @@
),
)
+
+# Remove the replacements below once
+# https://github.com/googleapis/synthtool/pull/1188 is merged
+
+# Update googleapis/repo-automation-bots repo to main in .kokoro/*.sh files
+s.replace(
+ ".kokoro/*.sh",
+ "repo-automation-bots/tree/master",
+ "repo-automation-bots/tree/main",
+)
+
+# Customize CONTRIBUTING.rst to replace master with main
+s.replace(
+ "CONTRIBUTING.rst",
+ "fetch and merge changes from upstream into master",
+ "fetch and merge changes from upstream into main",
+)
+
+s.replace(
+ "CONTRIBUTING.rst", "git merge upstream/master", "git merge upstream/main",
+)
+
+s.replace(
+ "CONTRIBUTING.rst",
+ """export GOOGLE_CLOUD_TESTING_BRANCH=\"master\"""",
+ """export GOOGLE_CLOUD_TESTING_BRANCH=\"main\"""",
+)
+
+s.replace(
+ "CONTRIBUTING.rst", r"remote \(``master``\)", "remote (``main``)",
+)
+
+s.replace(
+ "CONTRIBUTING.rst", "blob/master/CONTRIBUTING.rst", "blob/main/CONTRIBUTING.rst",
+)
+
+s.replace(
+ "CONTRIBUTING.rst", "blob/master/noxfile.py", "blob/main/noxfile.py",
+)
+
+s.replace(
+ "docs/conf.py", "master_doc", "root_doc",
+)
+
+s.replace(
+ "docs/conf.py", "# The master toctree document.", "# The root toctree document.",
+)
+
s.shell.run(["nox", "-s", "blacken"], hide_output=False)
diff --git a/samples/create_routine.py b/samples/create_routine.py
index 012c7927a..1cb4a80b4 100644
--- a/samples/create_routine.py
+++ b/samples/create_routine.py
@@ -22,7 +22,7 @@ def create_routine(routine_id):
# Construct a BigQuery client object.
client = bigquery.Client()
- # TODO(developer): Choose a fully-qualified ID for the routine.
+ # TODO(developer): Choose a fully qualified ID for the routine.
# routine_id = "my-project.my_dataset.my_routine"
routine = bigquery.Routine(
diff --git a/samples/geography/requirements-test.txt b/samples/geography/requirements-test.txt
index b0cf76724..5d836a5c5 100644
--- a/samples/geography/requirements-test.txt
+++ b/samples/geography/requirements-test.txt
@@ -1,2 +1,2 @@
-pytest==6.2.4
+pytest==6.2.5
mock==4.0.3
diff --git a/samples/geography/requirements.txt b/samples/geography/requirements.txt
index b5fe247cb..c325ee5e4 100644
--- a/samples/geography/requirements.txt
+++ b/samples/geography/requirements.txt
@@ -10,17 +10,17 @@ dataclasses==0.6; python_version < '3.7'
Fiona==1.8.20
geojson==2.5.0
geopandas==0.9.0
-google-api-core==1.31.2
-google-auth==1.35.0
-google-cloud-bigquery==2.25.0
+google-api-core==2.0.0
+google-auth==2.0.1
+google-cloud-bigquery==2.25.1
google-cloud-bigquery-storage==2.6.3
google-cloud-core==2.0.0
-google-crc32c==1.1.2
-google-resumable-media==1.3.3
+google-crc32c==1.1.3
+google-resumable-media==2.0.0
googleapis-common-protos==1.53.0
grpcio==1.39.0
idna==3.2
-importlib-metadata==4.6.4
+importlib-metadata==4.8.1
libcst==0.3.20
munch==2.5.0
mypy-extensions==0.4.3
@@ -36,7 +36,8 @@ pyasn1==0.4.8
pyasn1-modules==0.2.8
pycparser==2.20
pyparsing==2.4.7
-pyproj==3.0.1
+pyproj==3.0.1; python_version < "3.7"
+pyproj==3.1.0; python_version > "3.6"
python-dateutil==2.8.2
pytz==2021.1
PyYAML==5.4.1
@@ -44,7 +45,7 @@ requests==2.26.0
rsa==4.7.2
Shapely==1.7.1
six==1.16.0
-typing-extensions==3.10.0.0
+typing-extensions==3.10.0.2
typing-inspect==0.7.1
urllib3==1.26.6
zipp==3.5.0
diff --git a/samples/snippets/requirements-test.txt b/samples/snippets/requirements-test.txt
index b8dee50d0..caa48813a 100644
--- a/samples/snippets/requirements-test.txt
+++ b/samples/snippets/requirements-test.txt
@@ -1,3 +1,3 @@
-google-cloud-testutils==1.0.0
-pytest==6.2.4
+google-cloud-testutils==1.1.0
+pytest==6.2.5
mock==4.0.3
diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt
index d75c747fb..3b30ceebf 100644
--- a/samples/snippets/requirements.txt
+++ b/samples/snippets/requirements.txt
@@ -1,4 +1,4 @@
-google-cloud-bigquery==2.25.0
+google-cloud-bigquery==2.25.1
google-cloud-bigquery-storage==2.6.3
google-auth-oauthlib==0.4.5
grpcio==1.39.0
diff --git a/tests/unit/test__pandas_helpers.py b/tests/unit/test__pandas_helpers.py
index a9b0ae21f..f0975ef65 100644
--- a/tests/unit/test__pandas_helpers.py
+++ b/tests/unit/test__pandas_helpers.py
@@ -821,6 +821,41 @@ def test_dataframe_to_json_generator(module_under_test):
assert list(rows) == expected
+def test_dataframe_to_json_generator_repeated_field(module_under_test):
+ pytest.importorskip(
+ "pandas",
+ minversion=str(PANDAS_MINIUM_VERSION),
+ reason=(
+ f"Requires `pandas version >= {PANDAS_MINIUM_VERSION}` "
+ "which introduces pandas.NA"
+ ),
+ )
+
+ df_data = [
+ collections.OrderedDict(
+ [("repeated_col", [pandas.NA, 2, None, 4]), ("not_repeated_col", "first")]
+ ),
+ collections.OrderedDict(
+ [
+ ("repeated_col", ["a", "b", mock.sentinel.foo, "d"]),
+ ("not_repeated_col", "second"),
+ ]
+ ),
+ ]
+ dataframe = pandas.DataFrame(df_data)
+
+ rows = module_under_test.dataframe_to_json_generator(dataframe)
+
+ expected = [
+ {"repeated_col": [pandas.NA, 2, None, 4], "not_repeated_col": "first"},
+ {
+ "repeated_col": ["a", "b", mock.sentinel.foo, "d"],
+ "not_repeated_col": "second",
+ },
+ ]
+ assert list(rows) == expected
+
+
@pytest.mark.skipif(pandas is None, reason="Requires `pandas`")
def test_list_columns_and_indexes_with_named_index(module_under_test):
df_data = collections.OrderedDict(
@@ -882,7 +917,7 @@ def test_list_columns_and_indexes_with_multiindex(module_under_test):
def test_dataframe_to_bq_schema_dict_sequence(module_under_test):
df_data = collections.OrderedDict(
[
- ("str_column", [u"hello", u"world"]),
+ ("str_column", ["hello", "world"]),
("int_column", [42, 8]),
("bool_column", [True, False]),
]
@@ -1070,7 +1105,7 @@ def test_dataframe_to_arrow_dict_sequence_schema(module_under_test):
]
dataframe = pandas.DataFrame(
- {"field01": [u"hello", u"world"], "field02": [True, False]}
+ {"field01": ["hello", "world"], "field02": [True, False]}
)
arrow_table = module_under_test.dataframe_to_arrow(dataframe, dict_schema)
@@ -1139,8 +1174,8 @@ def test_dataframe_to_parquet_compression_method(module_under_test):
def test_dataframe_to_bq_schema_fallback_needed_wo_pyarrow(module_under_test):
dataframe = pandas.DataFrame(
data=[
- {"id": 10, "status": u"FOO", "execution_date": datetime.date(2019, 5, 10)},
- {"id": 20, "status": u"BAR", "created_at": datetime.date(2018, 9, 12)},
+ {"id": 10, "status": "FOO", "execution_date": datetime.date(2019, 5, 10)},
+ {"id": 20, "status": "BAR", "created_at": datetime.date(2018, 9, 12)},
]
)
@@ -1167,8 +1202,8 @@ def test_dataframe_to_bq_schema_fallback_needed_wo_pyarrow(module_under_test):
def test_dataframe_to_bq_schema_fallback_needed_w_pyarrow(module_under_test):
dataframe = pandas.DataFrame(
data=[
- {"id": 10, "status": u"FOO", "created_at": datetime.date(2019, 5, 10)},
- {"id": 20, "status": u"BAR", "created_at": datetime.date(2018, 9, 12)},
+ {"id": 10, "status": "FOO", "created_at": datetime.date(2019, 5, 10)},
+ {"id": 20, "status": "BAR", "created_at": datetime.date(2018, 9, 12)},
]
)
@@ -1197,8 +1232,8 @@ def test_dataframe_to_bq_schema_fallback_needed_w_pyarrow(module_under_test):
def test_dataframe_to_bq_schema_pyarrow_fallback_fails(module_under_test):
dataframe = pandas.DataFrame(
data=[
- {"struct_field": {"one": 2}, "status": u"FOO"},
- {"struct_field": {"two": u"222"}, "status": u"BAR"},
+ {"struct_field": {"one": 2}, "status": "FOO"},
+ {"struct_field": {"two": "222"}, "status": "BAR"},
]
)
@@ -1252,7 +1287,7 @@ def test_augment_schema_type_detection_succeeds(module_under_test):
"timestamp_field": datetime.datetime(2005, 5, 31, 14, 25, 55),
"date_field": datetime.date(2005, 5, 31),
"bytes_field": b"some bytes",
- "string_field": u"some characters",
+ "string_field": "some characters",
"numeric_field": decimal.Decimal("123.456"),
"bignumeric_field": decimal.Decimal("{d38}.{d38}".format(d38="9" * 38)),
}
@@ -1312,13 +1347,13 @@ def test_augment_schema_type_detection_fails(module_under_test):
dataframe = pandas.DataFrame(
data=[
{
- "status": u"FOO",
+ "status": "FOO",
"struct_field": {"one": 1},
- "struct_field_2": {"foo": u"123"},
+ "struct_field_2": {"foo": "123"},
},
{
- "status": u"BAR",
- "struct_field": {"two": u"111"},
+ "status": "BAR",
+ "struct_field": {"two": "111"},
"struct_field_2": {"bar": 27},
},
]
@@ -1351,7 +1386,7 @@ def test_dataframe_to_parquet_dict_sequence_schema(module_under_test):
]
dataframe = pandas.DataFrame(
- {"field01": [u"hello", u"world"], "field02": [True, False]}
+ {"field01": ["hello", "world"], "field02": [True, False]}
)
write_table_patch = mock.patch.object(
diff --git a/tests/unit/test_dbapi__helpers.py b/tests/unit/test_dbapi__helpers.py
index b33203354..5965a4817 100644
--- a/tests/unit/test_dbapi__helpers.py
+++ b/tests/unit/test_dbapi__helpers.py
@@ -612,8 +612,8 @@ def test_complex_query_parameter_type_errors(type_, value, expect):
"parameters,parameter_types,expect",
[
(
- [[], dict(name="ch1", bdate=datetime.date(2021, 1, 1))],
- ["ARRAY", "struct"],
+ [[], dict(name="ch1", b_date=datetime.date(2021, 1, 1))],
+ ["ARRAY", "struct"],
[
{
"parameterType": {"arrayType": {"type": "INT64"}, "type": "ARRAY"},
@@ -623,13 +623,13 @@ def test_complex_query_parameter_type_errors(type_, value, expect):
"parameterType": {
"structTypes": [
{"name": "name", "type": {"type": "STRING"}},
- {"name": "bdate", "type": {"type": "DATE"}},
+ {"name": "b_date", "type": {"type": "DATE"}},
],
"type": "STRUCT",
},
"parameterValue": {
"structValues": {
- "bdate": {"value": "2021-01-01"},
+ "b_date": {"value": "2021-01-01"},
"name": {"value": "ch1"},
}
},
diff --git a/tests/unit/test_dbapi_cursor.py b/tests/unit/test_dbapi_cursor.py
index 026810aaf..cb55da889 100644
--- a/tests/unit/test_dbapi_cursor.py
+++ b/tests/unit/test_dbapi_cursor.py
@@ -809,6 +809,10 @@ def test__format_operation_no_placeholders(self):
"values(%%%%%(foo:INT64)s, %(bar)s)",
("values(%%%%%(foo)s, %(bar)s)", dict(foo="INT64")),
),
+ (
+ "values(%%%%%(foo:struct)s, %(bar)s)",
+ ("values(%%%%%(foo)s, %(bar)s)", dict(foo="struct")),
+ ),
(
"values(%%%%%(foo:struct)s, %(bar)s)",
("values(%%%%%(foo)s, %(bar)s)", dict(foo="struct")),