From 5bbc47ec818f662a6099e494b360305bf8ef0481 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Thu, 18 Feb 2021 23:09:22 +0100 Subject: [PATCH 01/24] chore(deps): update dependency google-cloud-bigquery-storage to v2.3.0 (#145) --- samples/quickstart/requirements.txt | 2 +- samples/to_dataframe/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/samples/quickstart/requirements.txt b/samples/quickstart/requirements.txt index ddc998aa..9155b4b9 100644 --- a/samples/quickstart/requirements.txt +++ b/samples/quickstart/requirements.txt @@ -1,2 +1,2 @@ fastavro -google-cloud-bigquery-storage==2.2.1 +google-cloud-bigquery-storage==2.3.0 diff --git a/samples/to_dataframe/requirements.txt b/samples/to_dataframe/requirements.txt index 24a67b26..ce88d3d7 100644 --- a/samples/to_dataframe/requirements.txt +++ b/samples/to_dataframe/requirements.txt @@ -1,5 +1,5 @@ google-auth==1.27.0 -google-cloud-bigquery-storage==2.2.1 +google-cloud-bigquery-storage==2.3.0 google-cloud-bigquery==2.8.0 pyarrow==3.0.0 ipython==7.10.2; python_version > '3.0' From b214310c901bd82f07670add4450e06e9b1f3b1b Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Fri, 19 Feb 2021 19:52:49 +0100 Subject: [PATCH 02/24] chore(deps): update dependency google-cloud-bigquery to v2.9.0 (#144) --- samples/to_dataframe/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/to_dataframe/requirements.txt b/samples/to_dataframe/requirements.txt index ce88d3d7..074e0512 100644 --- a/samples/to_dataframe/requirements.txt +++ b/samples/to_dataframe/requirements.txt @@ -1,6 +1,6 @@ google-auth==1.27.0 google-cloud-bigquery-storage==2.3.0 -google-cloud-bigquery==2.8.0 +google-cloud-bigquery==2.9.0 pyarrow==3.0.0 ipython==7.10.2; python_version > '3.0' ipython==5.9.0; python_version < '3.0' From f1bc4128e099e83d5a15408bd6fd2129c2151301 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Thu, 25 Feb 2021 22:05:35 +0100 Subject: [PATCH 03/24] chore(deps): update dependency google-cloud-bigquery to v2.10.0 (#146) --- samples/to_dataframe/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/to_dataframe/requirements.txt b/samples/to_dataframe/requirements.txt index 074e0512..5cca9eb7 100644 --- a/samples/to_dataframe/requirements.txt +++ b/samples/to_dataframe/requirements.txt @@ -1,6 +1,6 @@ google-auth==1.27.0 google-cloud-bigquery-storage==2.3.0 -google-cloud-bigquery==2.9.0 +google-cloud-bigquery==2.10.0 pyarrow==3.0.0 ipython==7.10.2; python_version > '3.0' ipython==5.9.0; python_version < '3.0' From 05e662bd5574a7b8bf89de1fa3a5839796753691 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Tue, 9 Mar 2021 20:58:05 +0100 Subject: [PATCH 04/24] chore(deps): update dependency google-auth to v1.27.1 (#150) --- samples/to_dataframe/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/to_dataframe/requirements.txt b/samples/to_dataframe/requirements.txt index 5cca9eb7..0679da2e 100644 --- a/samples/to_dataframe/requirements.txt +++ b/samples/to_dataframe/requirements.txt @@ -1,4 +1,4 @@ -google-auth==1.27.0 +google-auth==1.27.1 google-cloud-bigquery-storage==2.3.0 google-cloud-bigquery==2.10.0 pyarrow==3.0.0 From a91c25d36e3021bca4efb473a4a4f31b1a93b5a5 Mon Sep 17 00:00:00 2001 From: Yoshi Automation Bot Date: Tue, 9 Mar 2021 12:03:22 -0800 Subject: [PATCH 05/24] test: install pyopenssl for mtls testing (#148) Source-Author: arithmetic1728 <58957152+arithmetic1728@users.noreply.github.com> Source-Date: Tue Mar 2 12:27:56 2021 -0800 Source-Repo: googleapis/synthtool Source-Sha: 0780323da96d5a53925fe0547757181fe76e8f1e Source-Link: https://github.com/googleapis/synthtool/commit/0780323da96d5a53925fe0547757181fe76e8f1e --- noxfile.py | 3 +++ synth.metadata | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/noxfile.py b/noxfile.py index c61b3fd6..6b6c4f55 100644 --- a/noxfile.py +++ b/noxfile.py @@ -123,6 +123,9 @@ def system(session): # Sanity check: Only run tests if the environment variable is set. if not os.environ.get("GOOGLE_APPLICATION_CREDENTIALS", ""): session.skip("Credentials must be set via environment variable") + # Install pyopenssl for mTLS testing. + if os.environ.get("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false") == "true": + session.install("pyopenssl") system_test_exists = os.path.exists(system_test_path) system_test_folder_exists = os.path.exists(system_test_folder_path) diff --git a/synth.metadata b/synth.metadata index 76b66ecd..9de488c9 100644 --- a/synth.metadata +++ b/synth.metadata @@ -4,7 +4,7 @@ "git": { "name": ".", "remote": "https://github.com/googleapis/python-bigquery-storage.git", - "sha": "7ac4df6b4c484768dc66e0afdc2cb658f0bb8a3d" + "sha": "f1bc4128e099e83d5a15408bd6fd2129c2151301" } }, { @@ -19,14 +19,14 @@ "git": { "name": "synthtool", "remote": "https://github.com/googleapis/synthtool.git", - "sha": "d17674372e27fb8f23013935e794aa37502071aa" + "sha": "0780323da96d5a53925fe0547757181fe76e8f1e" } }, { "git": { "name": "synthtool", "remote": "https://github.com/googleapis/synthtool.git", - "sha": "d17674372e27fb8f23013935e794aa37502071aa" + "sha": "0780323da96d5a53925fe0547757181fe76e8f1e" } } ], From ed0ea083ef091090f8c5461153c72be05b23c8e5 Mon Sep 17 00:00:00 2001 From: Yoshi Automation Bot Date: Tue, 9 Mar 2021 12:04:37 -0800 Subject: [PATCH 06/24] chore: update gapic-generator-python to 0.40.11 (#147) PiperOrigin-RevId: 359562873 Source-Author: Google APIs Source-Date: Thu Feb 25 10:52:32 2021 -0800 Source-Repo: googleapis/googleapis Source-Sha: 07932bb995e7dc91b43620ea8402c6668c7d102c Source-Link: https://github.com/googleapis/googleapis/commit/07932bb995e7dc91b43620ea8402c6668c7d102c --- .../services/big_query_read/async_client.py | 32 ++++++- .../services/big_query_read/async_client.py | 32 ++++++- .../services/big_query_write/async_client.py | 32 ++++++- synth.metadata | 4 +- .../gapic/bigquery_storage_v1/__init__.py | 15 +++ .../test_big_query_read.py | 60 +++++++++++- .../bigquery_storage_v1beta2/__init__.py | 15 +++ .../test_big_query_read.py | 60 +++++++++++- .../test_big_query_write.py | 96 ++++++++++++++++++- 9 files changed, 332 insertions(+), 14 deletions(-) diff --git a/google/cloud/bigquery_storage_v1/services/big_query_read/async_client.py b/google/cloud/bigquery_storage_v1/services/big_query_read/async_client.py index 5363e60f..a2a2fb21 100644 --- a/google/cloud/bigquery_storage_v1/services/big_query_read/async_client.py +++ b/google/cloud/bigquery_storage_v1/services/big_query_read/async_client.py @@ -81,8 +81,36 @@ class BigQueryReadAsyncClient: BigQueryReadClient.parse_common_location_path ) - from_service_account_info = BigQueryReadClient.from_service_account_info - from_service_account_file = BigQueryReadClient.from_service_account_file + @classmethod + def from_service_account_info(cls, info: dict, *args, **kwargs): + """Creates an instance of this client using the provided credentials info. + + Args: + info (dict): The service account private key info. + args: Additional arguments to pass to the constructor. + kwargs: Additional arguments to pass to the constructor. + + Returns: + BigQueryReadAsyncClient: The constructed client. + """ + return BigQueryReadClient.from_service_account_info.__func__(BigQueryReadAsyncClient, info, *args, **kwargs) # type: ignore + + @classmethod + def from_service_account_file(cls, filename: str, *args, **kwargs): + """Creates an instance of this client using the provided credentials + file. + + Args: + filename (str): The path to the service account private key json + file. + args: Additional arguments to pass to the constructor. + kwargs: Additional arguments to pass to the constructor. + + Returns: + BigQueryReadAsyncClient: The constructed client. + """ + return BigQueryReadClient.from_service_account_file.__func__(BigQueryReadAsyncClient, filename, *args, **kwargs) # type: ignore + from_service_account_json = from_service_account_file @property diff --git a/google/cloud/bigquery_storage_v1beta2/services/big_query_read/async_client.py b/google/cloud/bigquery_storage_v1beta2/services/big_query_read/async_client.py index 69b6ebe2..47bee37d 100644 --- a/google/cloud/bigquery_storage_v1beta2/services/big_query_read/async_client.py +++ b/google/cloud/bigquery_storage_v1beta2/services/big_query_read/async_client.py @@ -83,8 +83,36 @@ class BigQueryReadAsyncClient: BigQueryReadClient.parse_common_location_path ) - from_service_account_info = BigQueryReadClient.from_service_account_info - from_service_account_file = BigQueryReadClient.from_service_account_file + @classmethod + def from_service_account_info(cls, info: dict, *args, **kwargs): + """Creates an instance of this client using the provided credentials info. + + Args: + info (dict): The service account private key info. + args: Additional arguments to pass to the constructor. + kwargs: Additional arguments to pass to the constructor. + + Returns: + BigQueryReadAsyncClient: The constructed client. + """ + return BigQueryReadClient.from_service_account_info.__func__(BigQueryReadAsyncClient, info, *args, **kwargs) # type: ignore + + @classmethod + def from_service_account_file(cls, filename: str, *args, **kwargs): + """Creates an instance of this client using the provided credentials + file. + + Args: + filename (str): The path to the service account private key json + file. + args: Additional arguments to pass to the constructor. + kwargs: Additional arguments to pass to the constructor. + + Returns: + BigQueryReadAsyncClient: The constructed client. + """ + return BigQueryReadClient.from_service_account_file.__func__(BigQueryReadAsyncClient, filename, *args, **kwargs) # type: ignore + from_service_account_json = from_service_account_file @property diff --git a/google/cloud/bigquery_storage_v1beta2/services/big_query_write/async_client.py b/google/cloud/bigquery_storage_v1beta2/services/big_query_write/async_client.py index 378bd9ef..b3793a40 100644 --- a/google/cloud/bigquery_storage_v1beta2/services/big_query_write/async_client.py +++ b/google/cloud/bigquery_storage_v1beta2/services/big_query_write/async_client.py @@ -92,8 +92,36 @@ class BigQueryWriteAsyncClient: BigQueryWriteClient.parse_common_location_path ) - from_service_account_info = BigQueryWriteClient.from_service_account_info - from_service_account_file = BigQueryWriteClient.from_service_account_file + @classmethod + def from_service_account_info(cls, info: dict, *args, **kwargs): + """Creates an instance of this client using the provided credentials info. + + Args: + info (dict): The service account private key info. + args: Additional arguments to pass to the constructor. + kwargs: Additional arguments to pass to the constructor. + + Returns: + BigQueryWriteAsyncClient: The constructed client. + """ + return BigQueryWriteClient.from_service_account_info.__func__(BigQueryWriteAsyncClient, info, *args, **kwargs) # type: ignore + + @classmethod + def from_service_account_file(cls, filename: str, *args, **kwargs): + """Creates an instance of this client using the provided credentials + file. + + Args: + filename (str): The path to the service account private key json + file. + args: Additional arguments to pass to the constructor. + kwargs: Additional arguments to pass to the constructor. + + Returns: + BigQueryWriteAsyncClient: The constructed client. + """ + return BigQueryWriteClient.from_service_account_file.__func__(BigQueryWriteAsyncClient, filename, *args, **kwargs) # type: ignore + from_service_account_json = from_service_account_file @property diff --git a/synth.metadata b/synth.metadata index 9de488c9..6dc15e6e 100644 --- a/synth.metadata +++ b/synth.metadata @@ -11,8 +11,8 @@ "git": { "name": "googleapis", "remote": "https://github.com/googleapis/googleapis.git", - "sha": "20712b8fe95001b312f62c6c5f33e3e3ec92cfaf", - "internalRef": "354996675" + "sha": "07932bb995e7dc91b43620ea8402c6668c7d102c", + "internalRef": "359562873" } }, { diff --git a/tests/unit/gapic/bigquery_storage_v1/__init__.py b/tests/unit/gapic/bigquery_storage_v1/__init__.py index 8b137891..42ffdf2b 100644 --- a/tests/unit/gapic/bigquery_storage_v1/__init__.py +++ b/tests/unit/gapic/bigquery_storage_v1/__init__.py @@ -1 +1,16 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/tests/unit/gapic/bigquery_storage_v1/test_big_query_read.py b/tests/unit/gapic/bigquery_storage_v1/test_big_query_read.py index 96f2fd43..5f0ba847 100644 --- a/tests/unit/gapic/bigquery_storage_v1/test_big_query_read.py +++ b/tests/unit/gapic/bigquery_storage_v1/test_big_query_read.py @@ -86,15 +86,17 @@ def test__get_default_mtls_endpoint(): assert BigQueryReadClient._get_default_mtls_endpoint(non_googleapi) == non_googleapi -def test_big_query_read_client_from_service_account_info(): +@pytest.mark.parametrize("client_class", [BigQueryReadClient, BigQueryReadAsyncClient,]) +def test_big_query_read_client_from_service_account_info(client_class): creds = credentials.AnonymousCredentials() with mock.patch.object( service_account.Credentials, "from_service_account_info" ) as factory: factory.return_value = creds info = {"valid": True} - client = BigQueryReadClient.from_service_account_info(info) + client = client_class.from_service_account_info(info) assert client.transport._credentials == creds + assert isinstance(client, client_class) assert client.transport._host == "bigquerystorage.googleapis.com:443" @@ -108,9 +110,11 @@ def test_big_query_read_client_from_service_account_file(client_class): factory.return_value = creds client = client_class.from_service_account_file("dummy/file/path.json") assert client.transport._credentials == creds + assert isinstance(client, client_class) client = client_class.from_service_account_json("dummy/file/path.json") assert client.transport._credentials == creds + assert isinstance(client, client_class) assert client.transport._host == "bigquerystorage.googleapis.com:443" @@ -476,6 +480,24 @@ def test_create_read_session_from_dict(): test_create_read_session(request_type=dict) +def test_create_read_session_empty_call(): + # This test is a coverage failsafe to make sure that totally empty calls, + # i.e. request == None and no flattened fields passed, work. + client = BigQueryReadClient( + credentials=credentials.AnonymousCredentials(), transport="grpc", + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.create_read_session), "__call__" + ) as call: + client.create_read_session() + call.assert_called() + _, args, _ = call.mock_calls[0] + + assert args[0] == storage.CreateReadSessionRequest() + + @pytest.mark.asyncio async def test_create_read_session_async( transport: str = "grpc_asyncio", request_type=storage.CreateReadSessionRequest @@ -705,6 +727,22 @@ def test_read_rows_from_dict(): test_read_rows(request_type=dict) +def test_read_rows_empty_call(): + # This test is a coverage failsafe to make sure that totally empty calls, + # i.e. request == None and no flattened fields passed, work. + client = BigQueryReadClient( + credentials=credentials.AnonymousCredentials(), transport="grpc", + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object(type(client.transport.read_rows), "__call__") as call: + client.read_rows() + call.assert_called() + _, args, _ = call.mock_calls[0] + + assert args[0] == storage.ReadRowsRequest() + + @pytest.mark.asyncio async def test_read_rows_async( transport: str = "grpc_asyncio", request_type=storage.ReadRowsRequest @@ -901,6 +939,24 @@ def test_split_read_stream_from_dict(): test_split_read_stream(request_type=dict) +def test_split_read_stream_empty_call(): + # This test is a coverage failsafe to make sure that totally empty calls, + # i.e. request == None and no flattened fields passed, work. + client = BigQueryReadClient( + credentials=credentials.AnonymousCredentials(), transport="grpc", + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.split_read_stream), "__call__" + ) as call: + client.split_read_stream() + call.assert_called() + _, args, _ = call.mock_calls[0] + + assert args[0] == storage.SplitReadStreamRequest() + + @pytest.mark.asyncio async def test_split_read_stream_async( transport: str = "grpc_asyncio", request_type=storage.SplitReadStreamRequest diff --git a/tests/unit/gapic/bigquery_storage_v1beta2/__init__.py b/tests/unit/gapic/bigquery_storage_v1beta2/__init__.py index 8b137891..42ffdf2b 100644 --- a/tests/unit/gapic/bigquery_storage_v1beta2/__init__.py +++ b/tests/unit/gapic/bigquery_storage_v1beta2/__init__.py @@ -1 +1,16 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/tests/unit/gapic/bigquery_storage_v1beta2/test_big_query_read.py b/tests/unit/gapic/bigquery_storage_v1beta2/test_big_query_read.py index 47dd3dfc..12c49412 100644 --- a/tests/unit/gapic/bigquery_storage_v1beta2/test_big_query_read.py +++ b/tests/unit/gapic/bigquery_storage_v1beta2/test_big_query_read.py @@ -88,15 +88,17 @@ def test__get_default_mtls_endpoint(): assert BigQueryReadClient._get_default_mtls_endpoint(non_googleapi) == non_googleapi -def test_big_query_read_client_from_service_account_info(): +@pytest.mark.parametrize("client_class", [BigQueryReadClient, BigQueryReadAsyncClient,]) +def test_big_query_read_client_from_service_account_info(client_class): creds = credentials.AnonymousCredentials() with mock.patch.object( service_account.Credentials, "from_service_account_info" ) as factory: factory.return_value = creds info = {"valid": True} - client = BigQueryReadClient.from_service_account_info(info) + client = client_class.from_service_account_info(info) assert client.transport._credentials == creds + assert isinstance(client, client_class) assert client.transport._host == "bigquerystorage.googleapis.com:443" @@ -110,9 +112,11 @@ def test_big_query_read_client_from_service_account_file(client_class): factory.return_value = creds client = client_class.from_service_account_file("dummy/file/path.json") assert client.transport._credentials == creds + assert isinstance(client, client_class) client = client_class.from_service_account_json("dummy/file/path.json") assert client.transport._credentials == creds + assert isinstance(client, client_class) assert client.transport._host == "bigquerystorage.googleapis.com:443" @@ -478,6 +482,24 @@ def test_create_read_session_from_dict(): test_create_read_session(request_type=dict) +def test_create_read_session_empty_call(): + # This test is a coverage failsafe to make sure that totally empty calls, + # i.e. request == None and no flattened fields passed, work. + client = BigQueryReadClient( + credentials=credentials.AnonymousCredentials(), transport="grpc", + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.create_read_session), "__call__" + ) as call: + client.create_read_session() + call.assert_called() + _, args, _ = call.mock_calls[0] + + assert args[0] == storage.CreateReadSessionRequest() + + @pytest.mark.asyncio async def test_create_read_session_async( transport: str = "grpc_asyncio", request_type=storage.CreateReadSessionRequest @@ -707,6 +729,22 @@ def test_read_rows_from_dict(): test_read_rows(request_type=dict) +def test_read_rows_empty_call(): + # This test is a coverage failsafe to make sure that totally empty calls, + # i.e. request == None and no flattened fields passed, work. + client = BigQueryReadClient( + credentials=credentials.AnonymousCredentials(), transport="grpc", + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object(type(client.transport.read_rows), "__call__") as call: + client.read_rows() + call.assert_called() + _, args, _ = call.mock_calls[0] + + assert args[0] == storage.ReadRowsRequest() + + @pytest.mark.asyncio async def test_read_rows_async( transport: str = "grpc_asyncio", request_type=storage.ReadRowsRequest @@ -903,6 +941,24 @@ def test_split_read_stream_from_dict(): test_split_read_stream(request_type=dict) +def test_split_read_stream_empty_call(): + # This test is a coverage failsafe to make sure that totally empty calls, + # i.e. request == None and no flattened fields passed, work. + client = BigQueryReadClient( + credentials=credentials.AnonymousCredentials(), transport="grpc", + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.split_read_stream), "__call__" + ) as call: + client.split_read_stream() + call.assert_called() + _, args, _ = call.mock_calls[0] + + assert args[0] == storage.SplitReadStreamRequest() + + @pytest.mark.asyncio async def test_split_read_stream_async( transport: str = "grpc_asyncio", request_type=storage.SplitReadStreamRequest diff --git a/tests/unit/gapic/bigquery_storage_v1beta2/test_big_query_write.py b/tests/unit/gapic/bigquery_storage_v1beta2/test_big_query_write.py index 8d9112b9..8cfb4aaf 100644 --- a/tests/unit/gapic/bigquery_storage_v1beta2/test_big_query_write.py +++ b/tests/unit/gapic/bigquery_storage_v1beta2/test_big_query_write.py @@ -94,15 +94,19 @@ def test__get_default_mtls_endpoint(): ) -def test_big_query_write_client_from_service_account_info(): +@pytest.mark.parametrize( + "client_class", [BigQueryWriteClient, BigQueryWriteAsyncClient,] +) +def test_big_query_write_client_from_service_account_info(client_class): creds = credentials.AnonymousCredentials() with mock.patch.object( service_account.Credentials, "from_service_account_info" ) as factory: factory.return_value = creds info = {"valid": True} - client = BigQueryWriteClient.from_service_account_info(info) + client = client_class.from_service_account_info(info) assert client.transport._credentials == creds + assert isinstance(client, client_class) assert client.transport._host == "bigquerystorage.googleapis.com:443" @@ -118,9 +122,11 @@ def test_big_query_write_client_from_service_account_file(client_class): factory.return_value = creds client = client_class.from_service_account_file("dummy/file/path.json") assert client.transport._credentials == creds + assert isinstance(client, client_class) client = client_class.from_service_account_json("dummy/file/path.json") assert client.transport._credentials == creds + assert isinstance(client, client_class) assert client.transport._host == "bigquerystorage.googleapis.com:443" @@ -487,6 +493,24 @@ def test_create_write_stream_from_dict(): test_create_write_stream(request_type=dict) +def test_create_write_stream_empty_call(): + # This test is a coverage failsafe to make sure that totally empty calls, + # i.e. request == None and no flattened fields passed, work. + client = BigQueryWriteClient( + credentials=credentials.AnonymousCredentials(), transport="grpc", + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.create_write_stream), "__call__" + ) as call: + client.create_write_stream() + call.assert_called() + _, args, _ = call.mock_calls[0] + + assert args[0] == storage.CreateWriteStreamRequest() + + @pytest.mark.asyncio async def test_create_write_stream_async( transport: str = "grpc_asyncio", request_type=storage.CreateWriteStreamRequest @@ -777,6 +801,22 @@ def test_get_write_stream_from_dict(): test_get_write_stream(request_type=dict) +def test_get_write_stream_empty_call(): + # This test is a coverage failsafe to make sure that totally empty calls, + # i.e. request == None and no flattened fields passed, work. + client = BigQueryWriteClient( + credentials=credentials.AnonymousCredentials(), transport="grpc", + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object(type(client.transport.get_write_stream), "__call__") as call: + client.get_write_stream() + call.assert_called() + _, args, _ = call.mock_calls[0] + + assert args[0] == storage.GetWriteStreamRequest() + + @pytest.mark.asyncio async def test_get_write_stream_async( transport: str = "grpc_asyncio", request_type=storage.GetWriteStreamRequest @@ -970,6 +1010,24 @@ def test_finalize_write_stream_from_dict(): test_finalize_write_stream(request_type=dict) +def test_finalize_write_stream_empty_call(): + # This test is a coverage failsafe to make sure that totally empty calls, + # i.e. request == None and no flattened fields passed, work. + client = BigQueryWriteClient( + credentials=credentials.AnonymousCredentials(), transport="grpc", + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.finalize_write_stream), "__call__" + ) as call: + client.finalize_write_stream() + call.assert_called() + _, args, _ = call.mock_calls[0] + + assert args[0] == storage.FinalizeWriteStreamRequest() + + @pytest.mark.asyncio async def test_finalize_write_stream_async( transport: str = "grpc_asyncio", request_type=storage.FinalizeWriteStreamRequest @@ -1171,6 +1229,24 @@ def test_batch_commit_write_streams_from_dict(): test_batch_commit_write_streams(request_type=dict) +def test_batch_commit_write_streams_empty_call(): + # This test is a coverage failsafe to make sure that totally empty calls, + # i.e. request == None and no flattened fields passed, work. + client = BigQueryWriteClient( + credentials=credentials.AnonymousCredentials(), transport="grpc", + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.batch_commit_write_streams), "__call__" + ) as call: + client.batch_commit_write_streams() + call.assert_called() + _, args, _ = call.mock_calls[0] + + assert args[0] == storage.BatchCommitWriteStreamsRequest() + + @pytest.mark.asyncio async def test_batch_commit_write_streams_async( transport: str = "grpc_asyncio", request_type=storage.BatchCommitWriteStreamsRequest @@ -1368,6 +1444,22 @@ def test_flush_rows_from_dict(): test_flush_rows(request_type=dict) +def test_flush_rows_empty_call(): + # This test is a coverage failsafe to make sure that totally empty calls, + # i.e. request == None and no flattened fields passed, work. + client = BigQueryWriteClient( + credentials=credentials.AnonymousCredentials(), transport="grpc", + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object(type(client.transport.flush_rows), "__call__") as call: + client.flush_rows() + call.assert_called() + _, args, _ = call.mock_calls[0] + + assert args[0] == storage.FlushRowsRequest() + + @pytest.mark.asyncio async def test_flush_rows_async( transport: str = "grpc_asyncio", request_type=storage.FlushRowsRequest From f90883067b098273e5d81cf9cab23104aa054bc8 Mon Sep 17 00:00:00 2001 From: Yoshi Automation Bot Date: Wed, 10 Mar 2021 08:47:06 -0800 Subject: [PATCH 07/24] chore: updates to gapic generator (#153) * chore: upgrade gapic-generator-python to 0.42.2 PiperOrigin-RevId: 361662015 Source-Author: Google APIs Source-Date: Mon Mar 8 14:47:18 2021 -0800 Source-Repo: googleapis/googleapis Source-Sha: 28a591963253d52ce3a25a918cafbdd9928de8cf Source-Link: https://github.com/googleapis/googleapis/commit/28a591963253d52ce3a25a918cafbdd9928de8cf * build: use gapic-generator-typescript v1.2.11. Fixed IAM v1 library generation. Committer: @alexander-fenster PiperOrigin-RevId: 361676678 Source-Author: Google APIs Source-Date: Mon Mar 8 15:51:18 2021 -0800 Source-Repo: googleapis/googleapis Source-Sha: 3aeb3a70f66457a9e6b07caff841719bb9873b57 Source-Link: https://github.com/googleapis/googleapis/commit/3aeb3a70f66457a9e6b07caff841719bb9873b57 --- .../bigquery_storage_v1/types/__init__.py | 36 ++++---- .../types/__init__.py | 84 +++++++++---------- synth.metadata | 6 +- 3 files changed, 63 insertions(+), 63 deletions(-) diff --git a/google/cloud/bigquery_storage_v1/types/__init__.py b/google/cloud/bigquery_storage_v1/types/__init__.py index 14fc7096..ee89843a 100644 --- a/google/cloud/bigquery_storage_v1/types/__init__.py +++ b/google/cloud/bigquery_storage_v1/types/__init__.py @@ -16,43 +16,43 @@ # from .arrow import ( - ArrowSchema, ArrowRecordBatch, + ArrowSchema, ) from .avro import ( - AvroSchema, AvroRows, -) -from .stream import ( - DataFormat, - ReadSession, - ReadStream, - DataFormat, + AvroSchema, ) from .storage import ( CreateReadSessionRequest, ReadRowsRequest, - ThrottleState, - StreamStats, ReadRowsResponse, SplitReadStreamRequest, SplitReadStreamResponse, + StreamStats, + ThrottleState, +) +from .stream import ( + DataFormat, + ReadSession, + ReadStream, + DataFormat, ) __all__ = ( - "ArrowSchema", "ArrowRecordBatch", - "AvroSchema", + "ArrowSchema", "AvroRows", - "DataFormat", - "ReadSession", - "ReadStream", - "DataFormat", + "AvroSchema", "CreateReadSessionRequest", "ReadRowsRequest", - "ThrottleState", - "StreamStats", "ReadRowsResponse", "SplitReadStreamRequest", "SplitReadStreamResponse", + "StreamStats", + "ThrottleState", + "DataFormat", + "ReadSession", + "ReadStream", + "DataFormat", ) diff --git a/google/cloud/bigquery_storage_v1beta2/types/__init__.py b/google/cloud/bigquery_storage_v1beta2/types/__init__.py index ba1cac28..359fc964 100644 --- a/google/cloud/bigquery_storage_v1beta2/types/__init__.py +++ b/google/cloud/bigquery_storage_v1beta2/types/__init__.py @@ -16,81 +16,81 @@ # from .arrow import ( - ArrowSchema, ArrowRecordBatch, + ArrowSchema, ArrowSerializationOptions, ) from .avro import ( - AvroSchema, AvroRows, + AvroSchema, ) from .protobuf import ( - ProtoSchema, ProtoRows, -) -from .table import ( - TableSchema, - TableFieldSchema, -) -from .stream import ( - DataFormat, - ReadSession, - ReadStream, - WriteStream, - DataFormat, + ProtoSchema, ) from .storage import ( - CreateReadSessionRequest, - ReadRowsRequest, - ThrottleState, - StreamStats, - ReadRowsResponse, - SplitReadStreamRequest, - SplitReadStreamResponse, - CreateWriteStreamRequest, AppendRowsRequest, AppendRowsResponse, - GetWriteStreamRequest, BatchCommitWriteStreamsRequest, BatchCommitWriteStreamsResponse, + CreateReadSessionRequest, + CreateWriteStreamRequest, FinalizeWriteStreamRequest, FinalizeWriteStreamResponse, FlushRowsRequest, FlushRowsResponse, + GetWriteStreamRequest, + ReadRowsRequest, + ReadRowsResponse, + SplitReadStreamRequest, + SplitReadStreamResponse, StorageError, + StreamStats, + ThrottleState, +) +from .stream import ( + DataFormat, + ReadSession, + ReadStream, + WriteStream, + DataFormat, +) +from .table import ( + TableFieldSchema, + TableSchema, ) __all__ = ( - "ArrowSchema", "ArrowRecordBatch", + "ArrowSchema", "ArrowSerializationOptions", - "AvroSchema", "AvroRows", - "ProtoSchema", + "AvroSchema", "ProtoRows", - "TableSchema", - "TableFieldSchema", - "DataFormat", - "ReadSession", - "ReadStream", - "WriteStream", - "DataFormat", - "CreateReadSessionRequest", - "ReadRowsRequest", - "ThrottleState", - "StreamStats", - "ReadRowsResponse", - "SplitReadStreamRequest", - "SplitReadStreamResponse", - "CreateWriteStreamRequest", + "ProtoSchema", "AppendRowsRequest", "AppendRowsResponse", - "GetWriteStreamRequest", "BatchCommitWriteStreamsRequest", "BatchCommitWriteStreamsResponse", + "CreateReadSessionRequest", + "CreateWriteStreamRequest", "FinalizeWriteStreamRequest", "FinalizeWriteStreamResponse", "FlushRowsRequest", "FlushRowsResponse", + "GetWriteStreamRequest", + "ReadRowsRequest", + "ReadRowsResponse", + "SplitReadStreamRequest", + "SplitReadStreamResponse", "StorageError", + "StreamStats", + "ThrottleState", + "DataFormat", + "ReadSession", + "ReadStream", + "WriteStream", + "DataFormat", + "TableFieldSchema", + "TableSchema", ) diff --git a/synth.metadata b/synth.metadata index 6dc15e6e..b272bb08 100644 --- a/synth.metadata +++ b/synth.metadata @@ -4,15 +4,15 @@ "git": { "name": ".", "remote": "https://github.com/googleapis/python-bigquery-storage.git", - "sha": "f1bc4128e099e83d5a15408bd6fd2129c2151301" + "sha": "ed0ea083ef091090f8c5461153c72be05b23c8e5" } }, { "git": { "name": "googleapis", "remote": "https://github.com/googleapis/googleapis.git", - "sha": "07932bb995e7dc91b43620ea8402c6668c7d102c", - "internalRef": "359562873" + "sha": "3aeb3a70f66457a9e6b07caff841719bb9873b57", + "internalRef": "361676678" } }, { From 44adb1d3fe0b9d54a4872de9bee430b0b1865ad4 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Wed, 10 Mar 2021 21:42:28 +0100 Subject: [PATCH 08/24] chore(deps): update dependency google-cloud-bigquery to v2.11.0 (#152) --- samples/to_dataframe/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/to_dataframe/requirements.txt b/samples/to_dataframe/requirements.txt index 0679da2e..5f4eaa3e 100644 --- a/samples/to_dataframe/requirements.txt +++ b/samples/to_dataframe/requirements.txt @@ -1,6 +1,6 @@ google-auth==1.27.1 google-cloud-bigquery-storage==2.3.0 -google-cloud-bigquery==2.10.0 +google-cloud-bigquery==2.11.0 pyarrow==3.0.0 ipython==7.10.2; python_version > '3.0' ipython==5.9.0; python_version < '3.0' From a2bfa09daf85c7ec6cd4c0428290aebbc991ddc6 Mon Sep 17 00:00:00 2001 From: Yoshi Automation Bot Date: Wed, 10 Mar 2021 13:00:06 -0800 Subject: [PATCH 09/24] chore: exclude tarball from code generation (#149) This PR was generated using Autosynth. :rainbow: Synth log will be available here: https://source.cloud.google.com/results/invocations/afde7285-b349-475a-bfbd-1d975a7f9f5d/targets - [ ] To automatically regenerate this PR, check this box. --- synth.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synth.py b/synth.py index 896ba9b4..851d3a15 100644 --- a/synth.py +++ b/synth.py @@ -35,6 +35,7 @@ s.move( library, excludes=[ + "bigquery-storage-*-py.tar.gz", "docs/conf.py", "docs/index.rst", f"google/cloud/bigquery_storage_{version}/__init__.py", From ee614ae7e67bb3ebdb7fcd837ee90498f8e15097 Mon Sep 17 00:00:00 2001 From: Yoshi Automation Bot Date: Tue, 16 Mar 2021 09:29:13 -0700 Subject: [PATCH 10/24] chore: add pre-commit-config to renovate ignore paths (#156) Disable renovate PRs on the .pre-commit-config.yaml which is templated from synthtool. https://docs.renovatebot.com/configuration-options/#ignorepaths Source-Author: Bu Sun Kim <8822365+busunkim96@users.noreply.github.com> Source-Date: Mon Mar 15 09:05:39 2021 -0600 Source-Repo: googleapis/synthtool Source-Sha: 2c54c473779ea731128cea61a3a6c975a08a5378 Source-Link: https://github.com/googleapis/synthtool/commit/2c54c473779ea731128cea61a3a6c975a08a5378 --- renovate.json | 3 ++- synth.metadata | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/renovate.json b/renovate.json index 4fa94931..f08bc22c 100644 --- a/renovate.json +++ b/renovate.json @@ -1,5 +1,6 @@ { "extends": [ "config:base", ":preserveSemverRanges" - ] + ], + "ignorePaths": [".pre-commit-config.yaml"] } diff --git a/synth.metadata b/synth.metadata index b272bb08..b3b1ebdf 100644 --- a/synth.metadata +++ b/synth.metadata @@ -4,7 +4,7 @@ "git": { "name": ".", "remote": "https://github.com/googleapis/python-bigquery-storage.git", - "sha": "ed0ea083ef091090f8c5461153c72be05b23c8e5" + "sha": "a2bfa09daf85c7ec6cd4c0428290aebbc991ddc6" } }, { @@ -19,14 +19,14 @@ "git": { "name": "synthtool", "remote": "https://github.com/googleapis/synthtool.git", - "sha": "0780323da96d5a53925fe0547757181fe76e8f1e" + "sha": "2c54c473779ea731128cea61a3a6c975a08a5378" } }, { "git": { "name": "synthtool", "remote": "https://github.com/googleapis/synthtool.git", - "sha": "0780323da96d5a53925fe0547757181fe76e8f1e" + "sha": "2c54c473779ea731128cea61a3a6c975a08a5378" } } ], From 8022e9d9e4d3a4b3e9d65922f22600664a4352de Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Tue, 16 Mar 2021 12:48:42 -0500 Subject: [PATCH 11/24] test: use job metadata to avoid test flakes (#155) * test: add sleep to avoid test flakes * decrease waiting time and move to where it is most needed (at time capture) * use load job times instead --- tests/system/conftest.py | 2 +- tests/system/reader/test_reader.py | 38 +++++++++++++++++++++--------- 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/tests/system/conftest.py b/tests/system/conftest.py index dd42e736..9d476dcb 100644 --- a/tests/system/conftest.py +++ b/tests/system/conftest.py @@ -88,7 +88,7 @@ def dataset(project_id, bq_client): bq_client.delete_dataset(dataset, delete_contents=True) -@pytest.fixture(scope="session") +@pytest.fixture def table(project_id, dataset, bq_client): from google.cloud import bigquery diff --git a/tests/system/reader/test_reader.py b/tests/system/reader/test_reader.py index d0328041..92bfcc54 100644 --- a/tests/system/reader/test_reader.py +++ b/tests/system/reader/test_reader.py @@ -26,6 +26,9 @@ from google.cloud import bigquery +_TABLE_FORMAT = "projects/{}/datasets/{}/tables/{}" + + def _to_bq_table_ref(table_name_string, partition_suffix=""): """Converts protobuf table reference to bigquery table reference. @@ -177,25 +180,37 @@ def test_column_selection_read( assert sorted(row.keys()) == ["age", "first_name"] -def test_snapshot(client_and_types, project_id, table_with_data_ref, bq_client): +@pytest.mark.parametrize("data_format", ("AVRO", "ARROW")) +def test_snapshot(client_and_types, project_id, table, bq_client, data_format): client, types = client_and_types - before_new_data = dt.datetime.now(tz=dt.timezone.utc) + + # load original data into the table + original_data = [ + {"first_name": "OGFoo", "last_name": "Smith", "age": 44}, + {"first_name": "OGBar", "last_name": "Jones", "age": 33}, + ] + og_job = bq_client.load_table_from_json(original_data, table).result() + og_time = og_job.ended # load additional data into the table new_data = [ - {"first_name": "NewGuyFoo", "last_name": "Smith", "age": 46}, - {"first_name": "NewGuyBar", "last_name": "Jones", "age": 30}, + {"first_name": "NewFoo", "last_name": "Smiff", "age": 43}, + {"first_name": "NewBar", "last_name": "Jomes", "age": 34}, ] + new_job = bq_client.load_table_from_json(new_data, table).result() + new_time = new_job.ended - destination = _to_bq_table_ref(table_with_data_ref) - bq_client.load_table_from_json(new_data, destination).result() + # Because we want our snapshot to be between when we loaded the original + # data and when the new data was loaded, take the average of the two load + # job completion times. + before_new_data = og_time + ((new_time - og_time) / 2) # read data using the timestamp before the additional data load - + table_path = _TABLE_FORMAT.format(table.project, table.dataset_id, table.table_id) read_session = types.ReadSession() - read_session.table = table_with_data_ref + read_session.table = table_path read_session.table_modifiers.snapshot_time = before_new_data - read_session.data_format = types.DataFormat.AVRO + read_session.data_format = data_format session = client.create_read_session( request={ @@ -209,10 +224,11 @@ def test_snapshot(client_and_types, project_id, table_with_data_ref, bq_client): rows = list(client.read_rows(stream).rows(session)) # verify that only the data before the timestamp was returned - assert len(rows) == 5 # all initial records + assert len(rows) == 2 # all initial records for row in rows: - assert "NewGuy" not in row["first_name"] # no new records + assert "OG" in str(row["first_name"]) + assert "New" not in str(row["first_name"]) def test_column_partitioned_table( From 81155e4444a0352345326ff3bd1f601711d2fdda Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Wed, 17 Mar 2021 15:15:56 +0100 Subject: [PATCH 12/24] chore(deps): update dependency google-cloud-bigquery to v2.12.0 (#157) --- samples/to_dataframe/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/to_dataframe/requirements.txt b/samples/to_dataframe/requirements.txt index 5f4eaa3e..072ac02a 100644 --- a/samples/to_dataframe/requirements.txt +++ b/samples/to_dataframe/requirements.txt @@ -1,6 +1,6 @@ google-auth==1.27.1 google-cloud-bigquery-storage==2.3.0 -google-cloud-bigquery==2.11.0 +google-cloud-bigquery==2.12.0 pyarrow==3.0.0 ipython==7.10.2; python_version > '3.0' ipython==5.9.0; python_version < '3.0' From a137732509a4f1d584a00730a2a349de2e6b580a Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Wed, 17 Mar 2021 15:37:52 +0100 Subject: [PATCH 13/24] chore(deps): update dependency google-auth to v1.28.0 (#158) --- samples/to_dataframe/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/to_dataframe/requirements.txt b/samples/to_dataframe/requirements.txt index 072ac02a..29a45256 100644 --- a/samples/to_dataframe/requirements.txt +++ b/samples/to_dataframe/requirements.txt @@ -1,4 +1,4 @@ -google-auth==1.27.1 +google-auth==1.28.0 google-cloud-bigquery-storage==2.3.0 google-cloud-bigquery==2.12.0 pyarrow==3.0.0 From 0cbf160245faf33038eaee366c105c0774d2cd3c Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Wed, 17 Mar 2021 11:10:04 -0500 Subject: [PATCH 14/24] test: add retry for rate limiting of load jobs (#160) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to https://github.com/googleapis/python-bigquery-storage/pull/155 With 2 load jobs right in a row, we see to be hitting rate limiting. Add `retry_403` from `google-cloud-bigquery` tests, which seem to successfully work around this. Thank you for opening a Pull Request! Before submitting your PR, there are a few things you can do to make sure it goes smoothly: - [ ] Make sure to open an issue as a [bug/issue](https://github.com/googleapis/python-bigquery-storage/issues/new/choose) before writing your code! That way we can discuss the change, evaluate designs, and agree on the general idea - [ ] Ensure the tests and linter pass - [ ] Code coverage does not decrease (if any source code was changed) - [ ] Appropriate docs were updated (if necessary) Fixes #151 Fixes #161 🦕 --- tests/system/__init__.py | 13 +++++++++ tests/system/conftest.py | 46 +++++++++++++++++++----------- tests/system/helpers.py | 36 +++++++++++++++++++++++ tests/system/reader/__init__.py | 13 +++++++++ tests/system/reader/test_reader.py | 10 +++++-- 5 files changed, 99 insertions(+), 19 deletions(-) create mode 100644 tests/system/__init__.py create mode 100644 tests/system/helpers.py create mode 100644 tests/system/reader/__init__.py diff --git a/tests/system/__init__.py b/tests/system/__init__.py new file mode 100644 index 00000000..7e1ec16e --- /dev/null +++ b/tests/system/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/system/conftest.py b/tests/system/conftest.py index 9d476dcb..a18777dd 100644 --- a/tests/system/conftest.py +++ b/tests/system/conftest.py @@ -20,8 +20,10 @@ import pytest -_TABLE_FORMAT = "projects/{}/datasets/{}/tables/{}" +from . import helpers + +_TABLE_FORMAT = "projects/{}/datasets/{}/tables/{}" _ASSETS_DIR = os.path.join(os.path.abspath(os.path.dirname(__file__)), "assets") @@ -98,8 +100,10 @@ def table(project_id, dataset, bq_client): bigquery.SchemaField("age", "INTEGER", mode="NULLABLE"), ] - table_id = "{}.{}.{}".format(project_id, dataset.dataset_id, "users") - bq_table = bigquery.Table(table_id, schema=schema) + unique_suffix = str(uuid.uuid4()).replace("-", "_") + table_id = "users_" + unique_suffix + table_id_full = f"{project_id}.{dataset.dataset_id}.{table_id}" + bq_table = bigquery.Table(table_id_full, schema=schema) created_table = bq_client.create_table(bq_table) yield created_table @@ -154,7 +158,7 @@ def all_types_table_ref(project_id, dataset, bq_client): ) yield table_ref - bq_client.delete_table(created_table) + helpers.retry_403(bq_client.delete_table)(created_table, not_found_ok=True) @pytest.fixture @@ -182,7 +186,7 @@ def ingest_partition_table_ref(project_id, dataset, bq_client): ) yield table_ref - bq_client.delete_table(created_table) + helpers.retry_403(bq_client.delete_table)(created_table, not_found_ok=True) @pytest.fixture @@ -209,29 +213,39 @@ def col_partition_table_ref(project_id, dataset, bq_client): ) yield table_ref - bq_client.delete_table(created_table) + helpers.retry_403(bq_client.delete_table)(created_table, not_found_ok=True) @pytest.fixture -def table_with_data_ref(dataset, table, bq_client): +def table_with_data_ref(project_id, dataset, bq_client): from google.cloud import bigquery + unique_suffix = str(uuid.uuid4()).replace("-", "_") + table_id = "users_" + unique_suffix + table_id_full = f"{project_id}.{dataset.dataset_id}.{table_id}" + schema = [ + bigquery.SchemaField("first_name", "STRING", mode="NULLABLE"), + bigquery.SchemaField("last_name", "STRING", mode="NULLABLE"), + bigquery.SchemaField("age", "INTEGER", mode="NULLABLE"), + ] + job_config = bigquery.LoadJobConfig() job_config.source_format = bigquery.SourceFormat.CSV job_config.skip_leading_rows = 1 - job_config.schema = table.schema + job_config.schema = schema filename = os.path.join(_ASSETS_DIR, "people_data.csv") - with open(filename, "rb") as source_file: - job = bq_client.load_table_from_file(source_file, table, job_config=job_config) + def create_table(): + with open(filename, "rb") as source_file: + job = bq_client.load_table_from_file( + source_file, table_id_full, job_config=job_config + ) + job.result() # wait for the load to complete - job.result() # wait for the load to complete + helpers.retry_403(create_table)() - table_ref = _TABLE_FORMAT.format(table.project, table.dataset_id, table.table_id) + table_ref = _TABLE_FORMAT.format(project_id, dataset.dataset_id, table_id) yield table_ref - # truncate table data - query = "DELETE FROM {}.{} WHERE 1 = 1".format(dataset.dataset_id, table.table_id) - query_job = bq_client.query(query, location="US") - query_job.result() + helpers.retry_403(bq_client.delete_table)(table_id_full, not_found_ok=True) diff --git a/tests/system/helpers.py b/tests/system/helpers.py new file mode 100644 index 00000000..8a0e862e --- /dev/null +++ b/tests/system/helpers.py @@ -0,0 +1,36 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Test utilities. + +Copied from the BigQuery client library +https://github.com/googleapis/python-bigquery/blob/master/tests/system/helpers.py +""" + +import google.api_core.exceptions +import test_utils.retry + + +def _rate_limit_exceeded(forbidden): + """Predicate: pass only exceptions with 'rateLimitExceeded' as reason.""" + return any(error["reason"] == "rateLimitExceeded" for error in forbidden._errors) + + +# We need to wait to stay within the rate limits. +# The alternative outcome is a 403 Forbidden response from upstream, which +# they return instead of the more appropriate 429. +# See https://cloud.google.com/bigquery/quota-policy +retry_403 = test_utils.retry.RetryErrors( + google.api_core.exceptions.Forbidden, error_predicate=_rate_limit_exceeded, +) diff --git a/tests/system/reader/__init__.py b/tests/system/reader/__init__.py new file mode 100644 index 00000000..7e1ec16e --- /dev/null +++ b/tests/system/reader/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/system/reader/test_reader.py b/tests/system/reader/test_reader.py index 92bfcc54..aae3f6ec 100644 --- a/tests/system/reader/test_reader.py +++ b/tests/system/reader/test_reader.py @@ -20,10 +20,11 @@ import decimal import re +from google.cloud import bigquery import pytest import pytz -from google.cloud import bigquery +from .. import helpers _TABLE_FORMAT = "projects/{}/datasets/{}/tables/{}" @@ -184,12 +185,15 @@ def test_column_selection_read( def test_snapshot(client_and_types, project_id, table, bq_client, data_format): client, types = client_and_types + def load_json(data): + return bq_client.load_table_from_json(data, table).result() + # load original data into the table original_data = [ {"first_name": "OGFoo", "last_name": "Smith", "age": 44}, {"first_name": "OGBar", "last_name": "Jones", "age": 33}, ] - og_job = bq_client.load_table_from_json(original_data, table).result() + og_job = helpers.retry_403(load_json)(original_data) og_time = og_job.ended # load additional data into the table @@ -197,7 +201,7 @@ def test_snapshot(client_and_types, project_id, table, bq_client, data_format): {"first_name": "NewFoo", "last_name": "Smiff", "age": 43}, {"first_name": "NewBar", "last_name": "Jomes", "age": 34}, ] - new_job = bq_client.load_table_from_json(new_data, table).result() + new_job = helpers.retry_403(load_json)(new_data) new_time = new_job.ended # Because we want our snapshot to be between when we loaded the original From 5a235c5cd194f8ca2fa7591032ba2cafd50a7dbc Mon Sep 17 00:00:00 2001 From: Yoshi Automation Bot Date: Wed, 17 Mar 2021 11:22:03 -0700 Subject: [PATCH 15/24] build(python): fail nox sessions if a python version is missing (#159) This PR was generated using Autosynth. :rainbow: Synth log will be available here: https://source.cloud.google.com/results/invocations/f0eeb0ef-c8e9-4fea-9a7f-b5ca7baded13/targets - [ ] To automatically regenerate this PR, check this box. Source-Link: https://github.com/googleapis/synthtool/commit/eda422b90c3dde4a872a13e6b78a8f802c40d0db --- noxfile.py | 3 +++ synth.metadata | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/noxfile.py b/noxfile.py index 6b6c4f55..0647537d 100644 --- a/noxfile.py +++ b/noxfile.py @@ -41,6 +41,9 @@ "docs", ] +# Error if a python version is missing +nox.options.error_on_missing_interpreters = True + @nox.session(python=DEFAULT_PYTHON_VERSION) def lint(session): diff --git a/synth.metadata b/synth.metadata index b3b1ebdf..b8417ae4 100644 --- a/synth.metadata +++ b/synth.metadata @@ -4,7 +4,7 @@ "git": { "name": ".", "remote": "https://github.com/googleapis/python-bigquery-storage.git", - "sha": "a2bfa09daf85c7ec6cd4c0428290aebbc991ddc6" + "sha": "8022e9d9e4d3a4b3e9d65922f22600664a4352de" } }, { @@ -19,14 +19,14 @@ "git": { "name": "synthtool", "remote": "https://github.com/googleapis/synthtool.git", - "sha": "2c54c473779ea731128cea61a3a6c975a08a5378" + "sha": "eda422b90c3dde4a872a13e6b78a8f802c40d0db" } }, { "git": { "name": "synthtool", "remote": "https://github.com/googleapis/synthtool.git", - "sha": "2c54c473779ea731128cea61a3a6c975a08a5378" + "sha": "eda422b90c3dde4a872a13e6b78a8f802c40d0db" } } ], From 2d981460e62960f273ceb2537a70aeda8cbde5ad Mon Sep 17 00:00:00 2001 From: Yoshi Automation Bot Date: Mon, 22 Mar 2021 07:43:52 -0700 Subject: [PATCH 16/24] chore(python): add kokoro configs for periodic builds against head (#162) This change should be non-destructive. Note for library repo maintainers: After applying this change, you can easily add (or change) periodic builds against head by adding config files in google3. See python-pubsub repo for example. Source-Author: Takashi Matsuo Source-Date: Fri Mar 19 11:17:59 2021 -0700 Source-Repo: googleapis/synthtool Source-Sha: 79c8dd7ee768292f933012d3a69a5b4676404cda Source-Link: https://github.com/googleapis/synthtool/commit/79c8dd7ee768292f933012d3a69a5b4676404cda --- .kokoro/samples/python3.6/periodic-head.cfg | 11 +++ .kokoro/samples/python3.7/periodic-head.cfg | 11 +++ .kokoro/samples/python3.8/periodic-head.cfg | 11 +++ .kokoro/test-samples-against-head.sh | 28 ++++++ .kokoro/test-samples-impl.sh | 102 ++++++++++++++++++++ .kokoro/test-samples.sh | 96 +++--------------- synth.metadata | 11 ++- 7 files changed, 187 insertions(+), 83 deletions(-) create mode 100644 .kokoro/samples/python3.6/periodic-head.cfg create mode 100644 .kokoro/samples/python3.7/periodic-head.cfg create mode 100644 .kokoro/samples/python3.8/periodic-head.cfg create mode 100755 .kokoro/test-samples-against-head.sh create mode 100755 .kokoro/test-samples-impl.sh diff --git a/.kokoro/samples/python3.6/periodic-head.cfg b/.kokoro/samples/python3.6/periodic-head.cfg new file mode 100644 index 00000000..f9cfcd33 --- /dev/null +++ b/.kokoro/samples/python3.6/periodic-head.cfg @@ -0,0 +1,11 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-pubsub/.kokoro/test-samples-against-head.sh" +} diff --git a/.kokoro/samples/python3.7/periodic-head.cfg b/.kokoro/samples/python3.7/periodic-head.cfg new file mode 100644 index 00000000..f9cfcd33 --- /dev/null +++ b/.kokoro/samples/python3.7/periodic-head.cfg @@ -0,0 +1,11 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-pubsub/.kokoro/test-samples-against-head.sh" +} diff --git a/.kokoro/samples/python3.8/periodic-head.cfg b/.kokoro/samples/python3.8/periodic-head.cfg new file mode 100644 index 00000000..f9cfcd33 --- /dev/null +++ b/.kokoro/samples/python3.8/periodic-head.cfg @@ -0,0 +1,11 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-pubsub/.kokoro/test-samples-against-head.sh" +} diff --git a/.kokoro/test-samples-against-head.sh b/.kokoro/test-samples-against-head.sh new file mode 100755 index 00000000..36ba35b5 --- /dev/null +++ b/.kokoro/test-samples-against-head.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# A customized test runner for samples. +# +# For periodic builds, you can specify this file for testing against head. + +# `-e` enables the script to automatically fail when a command fails +# `-o pipefail` sets the exit code to the rightmost comment to exit with a non-zero +set -eo pipefail +# Enables `**` to include files nested inside sub-folders +shopt -s globstar + +cd github/python-bigquery-storage + +exec .kokoro/test-samples-impl.sh diff --git a/.kokoro/test-samples-impl.sh b/.kokoro/test-samples-impl.sh new file mode 100755 index 00000000..cf5de74c --- /dev/null +++ b/.kokoro/test-samples-impl.sh @@ -0,0 +1,102 @@ +#!/bin/bash +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# `-e` enables the script to automatically fail when a command fails +# `-o pipefail` sets the exit code to the rightmost comment to exit with a non-zero +set -eo pipefail +# Enables `**` to include files nested inside sub-folders +shopt -s globstar + +# Exit early if samples directory doesn't exist +if [ ! -d "./samples" ]; then + echo "No tests run. `./samples` not found" + exit 0 +fi + +# Disable buffering, so that the logs stream through. +export PYTHONUNBUFFERED=1 + +# Debug: show build environment +env | grep KOKORO + +# Install nox +python3.6 -m pip install --upgrade --quiet nox + +# Use secrets acessor service account to get secrets +if [[ -f "${KOKORO_GFILE_DIR}/secrets_viewer_service_account.json" ]]; then + gcloud auth activate-service-account \ + --key-file="${KOKORO_GFILE_DIR}/secrets_viewer_service_account.json" \ + --project="cloud-devrel-kokoro-resources" +fi + +# This script will create 3 files: +# - testing/test-env.sh +# - testing/service-account.json +# - testing/client-secrets.json +./scripts/decrypt-secrets.sh + +source ./testing/test-env.sh +export GOOGLE_APPLICATION_CREDENTIALS=$(pwd)/testing/service-account.json + +# For cloud-run session, we activate the service account for gcloud sdk. +gcloud auth activate-service-account \ + --key-file "${GOOGLE_APPLICATION_CREDENTIALS}" + +export GOOGLE_CLIENT_SECRETS=$(pwd)/testing/client-secrets.json + +echo -e "\n******************** TESTING PROJECTS ********************" + +# Switch to 'fail at end' to allow all tests to complete before exiting. +set +e +# Use RTN to return a non-zero value if the test fails. +RTN=0 +ROOT=$(pwd) +# Find all requirements.txt in the samples directory (may break on whitespace). +for file in samples/**/requirements.txt; do + cd "$ROOT" + # Navigate to the project folder. + file=$(dirname "$file") + cd "$file" + + echo "------------------------------------------------------------" + echo "- testing $file" + echo "------------------------------------------------------------" + + # Use nox to execute the tests for the project. + python3.6 -m nox -s "$RUN_TESTS_SESSION" + EXIT=$? + + # If this is a periodic build, send the test log to the FlakyBot. + # See https://github.com/googleapis/repo-automation-bots/tree/master/packages/flakybot. + if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"periodic"* ]]; then + chmod +x $KOKORO_GFILE_DIR/linux_amd64/flakybot + $KOKORO_GFILE_DIR/linux_amd64/flakybot + fi + + if [[ $EXIT -ne 0 ]]; then + RTN=1 + echo -e "\n Testing failed: Nox returned a non-zero exit code. \n" + else + echo -e "\n Testing completed.\n" + fi + +done +cd "$ROOT" + +# Workaround for Kokoro permissions issue: delete secrets +rm testing/{test-env.sh,client-secrets.json,service-account.json} + +exit "$RTN" diff --git a/.kokoro/test-samples.sh b/.kokoro/test-samples.sh index 056e11a3..3ef99a86 100755 --- a/.kokoro/test-samples.sh +++ b/.kokoro/test-samples.sh @@ -13,6 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +# The default test runner for samples. +# +# For periodic builds, we rewinds the repo to the latest release, and +# run test-samples-impl.sh. # `-e` enables the script to automatically fail when a command fails # `-o pipefail` sets the exit code to the rightmost comment to exit with a non-zero @@ -24,87 +28,19 @@ cd github/python-bigquery-storage # Run periodic samples tests at latest release if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"periodic"* ]]; then + # preserving the test runner implementation. + cp .kokoro/test-samples-impl.sh "${TMPDIR}/test-samples-impl.sh" + echo "--- IMPORTANT IMPORTANT IMPORTANT ---" + echo "Now we rewind the repo back to the latest release..." LATEST_RELEASE=$(git describe --abbrev=0 --tags) git checkout $LATEST_RELEASE -fi - -# Exit early if samples directory doesn't exist -if [ ! -d "./samples" ]; then - echo "No tests run. `./samples` not found" - exit 0 -fi - -# Disable buffering, so that the logs stream through. -export PYTHONUNBUFFERED=1 - -# Debug: show build environment -env | grep KOKORO - -# Install nox -python3.6 -m pip install --upgrade --quiet nox - -# Use secrets acessor service account to get secrets -if [[ -f "${KOKORO_GFILE_DIR}/secrets_viewer_service_account.json" ]]; then - gcloud auth activate-service-account \ - --key-file="${KOKORO_GFILE_DIR}/secrets_viewer_service_account.json" \ - --project="cloud-devrel-kokoro-resources" -fi - -# This script will create 3 files: -# - testing/test-env.sh -# - testing/service-account.json -# - testing/client-secrets.json -./scripts/decrypt-secrets.sh - -source ./testing/test-env.sh -export GOOGLE_APPLICATION_CREDENTIALS=$(pwd)/testing/service-account.json - -# For cloud-run session, we activate the service account for gcloud sdk. -gcloud auth activate-service-account \ - --key-file "${GOOGLE_APPLICATION_CREDENTIALS}" - -export GOOGLE_CLIENT_SECRETS=$(pwd)/testing/client-secrets.json - -echo -e "\n******************** TESTING PROJECTS ********************" - -# Switch to 'fail at end' to allow all tests to complete before exiting. -set +e -# Use RTN to return a non-zero value if the test fails. -RTN=0 -ROOT=$(pwd) -# Find all requirements.txt in the samples directory (may break on whitespace). -for file in samples/**/requirements.txt; do - cd "$ROOT" - # Navigate to the project folder. - file=$(dirname "$file") - cd "$file" - - echo "------------------------------------------------------------" - echo "- testing $file" - echo "------------------------------------------------------------" - - # Use nox to execute the tests for the project. - python3.6 -m nox -s "$RUN_TESTS_SESSION" - EXIT=$? - - # If this is a periodic build, send the test log to the FlakyBot. - # See https://github.com/googleapis/repo-automation-bots/tree/master/packages/flakybot. - if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"periodic"* ]]; then - chmod +x $KOKORO_GFILE_DIR/linux_amd64/flakybot - $KOKORO_GFILE_DIR/linux_amd64/flakybot + echo "The current head is: " + echo $(git rev-parse --verify HEAD) + echo "--- IMPORTANT IMPORTANT IMPORTANT ---" + # move back the test runner implementation if there's no file. + if [ ! -f .kokoro/test-samples-impl.sh ]; then + cp "${TMPDIR}/test-samples-impl.sh" .kokoro/test-samples-impl.sh fi +fi - if [[ $EXIT -ne 0 ]]; then - RTN=1 - echo -e "\n Testing failed: Nox returned a non-zero exit code. \n" - else - echo -e "\n Testing completed.\n" - fi - -done -cd "$ROOT" - -# Workaround for Kokoro permissions issue: delete secrets -rm testing/{test-env.sh,client-secrets.json,service-account.json} - -exit "$RTN" +exec .kokoro/test-samples-impl.sh diff --git a/synth.metadata b/synth.metadata index b8417ae4..f30677e6 100644 --- a/synth.metadata +++ b/synth.metadata @@ -4,7 +4,7 @@ "git": { "name": ".", "remote": "https://github.com/googleapis/python-bigquery-storage.git", - "sha": "8022e9d9e4d3a4b3e9d65922f22600664a4352de" + "sha": "5a235c5cd194f8ca2fa7591032ba2cafd50a7dbc" } }, { @@ -19,14 +19,14 @@ "git": { "name": "synthtool", "remote": "https://github.com/googleapis/synthtool.git", - "sha": "eda422b90c3dde4a872a13e6b78a8f802c40d0db" + "sha": "79c8dd7ee768292f933012d3a69a5b4676404cda" } }, { "git": { "name": "synthtool", "remote": "https://github.com/googleapis/synthtool.git", - "sha": "eda422b90c3dde4a872a13e6b78a8f802c40d0db" + "sha": "79c8dd7ee768292f933012d3a69a5b4676404cda" } } ], @@ -83,16 +83,21 @@ ".kokoro/samples/lint/presubmit.cfg", ".kokoro/samples/python3.6/common.cfg", ".kokoro/samples/python3.6/continuous.cfg", + ".kokoro/samples/python3.6/periodic-head.cfg", ".kokoro/samples/python3.6/periodic.cfg", ".kokoro/samples/python3.6/presubmit.cfg", ".kokoro/samples/python3.7/common.cfg", ".kokoro/samples/python3.7/continuous.cfg", + ".kokoro/samples/python3.7/periodic-head.cfg", ".kokoro/samples/python3.7/periodic.cfg", ".kokoro/samples/python3.7/presubmit.cfg", ".kokoro/samples/python3.8/common.cfg", ".kokoro/samples/python3.8/continuous.cfg", + ".kokoro/samples/python3.8/periodic-head.cfg", ".kokoro/samples/python3.8/periodic.cfg", ".kokoro/samples/python3.8/presubmit.cfg", + ".kokoro/test-samples-against-head.sh", + ".kokoro/test-samples-impl.sh", ".kokoro/test-samples.sh", ".kokoro/trampoline.sh", ".kokoro/trampoline_v2.sh", From 8c1071337ea6c76a61f601b46254e6f07d8f5add Mon Sep 17 00:00:00 2001 From: Yoshi Automation Bot Date: Tue, 23 Mar 2021 14:01:25 -0700 Subject: [PATCH 17/24] chore: upgrade gapic-generator-python to 0.43.1 (#164) PiperOrigin-RevId: 364411656 Source-Author: Google APIs Source-Date: Mon Mar 22 14:40:22 2021 -0700 Source-Repo: googleapis/googleapis Source-Sha: 149a3a84c29c9b8189576c7442ccb6dcf6a8f95b Source-Link: https://github.com/googleapis/googleapis/commit/149a3a84c29c9b8189576c7442ccb6dcf6a8f95b --- .../services/big_query_read/async_client.py | 3 + .../big_query_read/transports/base.py | 21 ++-- .../big_query_read/transports/grpc.py | 101 ++++++---------- .../big_query_read/transports/grpc_asyncio.py | 109 +++++++----------- .../services/big_query_read/async_client.py | 3 + .../big_query_read/transports/base.py | 21 ++-- .../big_query_read/transports/grpc.py | 101 ++++++---------- .../big_query_read/transports/grpc_asyncio.py | 109 +++++++----------- .../services/big_query_write/async_client.py | 6 + .../big_query_write/transports/base.py | 24 ++-- .../big_query_write/transports/grpc.py | 101 ++++++---------- .../transports/grpc_asyncio.py | 109 +++++++----------- synth.metadata | 6 +- 13 files changed, 288 insertions(+), 426 deletions(-) diff --git a/google/cloud/bigquery_storage_v1/services/big_query_read/async_client.py b/google/cloud/bigquery_storage_v1/services/big_query_read/async_client.py index a2a2fb21..c8c28dab 100644 --- a/google/cloud/bigquery_storage_v1/services/big_query_read/async_client.py +++ b/google/cloud/bigquery_storage_v1/services/big_query_read/async_client.py @@ -288,6 +288,7 @@ async def create_read_session( predicate=retries.if_exception_type( exceptions.DeadlineExceeded, exceptions.ServiceUnavailable, ), + deadline=600.0, ), default_timeout=600.0, client_info=DEFAULT_CLIENT_INFO, @@ -386,6 +387,7 @@ def read_rows( maximum=60.0, multiplier=1.3, predicate=retries.if_exception_type(exceptions.ServiceUnavailable,), + deadline=86400.0, ), default_timeout=86400.0, client_info=DEFAULT_CLIENT_INFO, @@ -457,6 +459,7 @@ async def split_read_stream( predicate=retries.if_exception_type( exceptions.DeadlineExceeded, exceptions.ServiceUnavailable, ), + deadline=600.0, ), default_timeout=600.0, client_info=DEFAULT_CLIENT_INFO, diff --git a/google/cloud/bigquery_storage_v1/services/big_query_read/transports/base.py b/google/cloud/bigquery_storage_v1/services/big_query_read/transports/base.py index 4497158f..991260bd 100644 --- a/google/cloud/bigquery_storage_v1/services/big_query_read/transports/base.py +++ b/google/cloud/bigquery_storage_v1/services/big_query_read/transports/base.py @@ -74,10 +74,10 @@ def __init__( scope (Optional[Sequence[str]]): A list of scopes. quota_project_id (Optional[str]): An optional project to use for billing and quota. - client_info (google.api_core.gapic_v1.client_info.ClientInfo): - The client info used to send a user-agent string along with - API requests. If ``None``, then default info will be used. - Generally, you only need to set this if you're developing + client_info (google.api_core.gapic_v1.client_info.ClientInfo): + The client info used to send a user-agent string along with + API requests. If ``None``, then default info will be used. + Generally, you only need to set this if you're developing your own client library. """ # Save the hostname. Default to port 443 (HTTPS) if none is specified. @@ -85,6 +85,9 @@ def __init__( host += ":443" self._host = host + # Save the scopes. + self._scopes = scopes or self.AUTH_SCOPES + # If no credentials are provided, then determine the appropriate # defaults. if credentials and credentials_file: @@ -94,20 +97,17 @@ def __init__( if credentials_file is not None: credentials, _ = auth.load_credentials_from_file( - credentials_file, scopes=scopes, quota_project_id=quota_project_id + credentials_file, scopes=self._scopes, quota_project_id=quota_project_id ) elif credentials is None: credentials, _ = auth.default( - scopes=scopes, quota_project_id=quota_project_id + scopes=self._scopes, quota_project_id=quota_project_id ) # Save the credentials. self._credentials = credentials - # Lifted into its own function so it can be stubbed out during tests. - self._prep_wrapped_messages(client_info) - def _prep_wrapped_messages(self, client_info): # Precompute the wrapped methods. self._wrapped_methods = { @@ -120,6 +120,7 @@ def _prep_wrapped_messages(self, client_info): predicate=retries.if_exception_type( exceptions.DeadlineExceeded, exceptions.ServiceUnavailable, ), + deadline=600.0, ), default_timeout=600.0, client_info=client_info, @@ -131,6 +132,7 @@ def _prep_wrapped_messages(self, client_info): maximum=60.0, multiplier=1.3, predicate=retries.if_exception_type(exceptions.ServiceUnavailable,), + deadline=86400.0, ), default_timeout=86400.0, client_info=client_info, @@ -144,6 +146,7 @@ def _prep_wrapped_messages(self, client_info): predicate=retries.if_exception_type( exceptions.DeadlineExceeded, exceptions.ServiceUnavailable, ), + deadline=600.0, ), default_timeout=600.0, client_info=client_info, diff --git a/google/cloud/bigquery_storage_v1/services/big_query_read/transports/grpc.py b/google/cloud/bigquery_storage_v1/services/big_query_read/transports/grpc.py index cfb3dbf8..4c7369e9 100644 --- a/google/cloud/bigquery_storage_v1/services/big_query_read/transports/grpc.py +++ b/google/cloud/bigquery_storage_v1/services/big_query_read/transports/grpc.py @@ -108,7 +108,9 @@ def __init__( google.api_core.exceptions.DuplicateCredentialArgs: If both ``credentials`` and ``credentials_file`` are passed. """ + self._grpc_channel = None self._ssl_channel_credentials = ssl_channel_credentials + self._stubs: Dict[str, Callable] = {} if api_mtls_endpoint: warnings.warn("api_mtls_endpoint is deprecated", DeprecationWarning) @@ -116,70 +118,50 @@ def __init__( warnings.warn("client_cert_source is deprecated", DeprecationWarning) if channel: - # Sanity check: Ensure that channel and credentials are not both - # provided. + # Ignore credentials if a channel was passed. credentials = False - # If a channel was explicitly provided, set it. self._grpc_channel = channel self._ssl_channel_credentials = None - elif api_mtls_endpoint: - host = ( - api_mtls_endpoint - if ":" in api_mtls_endpoint - else api_mtls_endpoint + ":443" - ) - - if credentials is None: - credentials, _ = auth.default( - scopes=self.AUTH_SCOPES, quota_project_id=quota_project_id - ) - - # Create SSL credentials with client_cert_source or application - # default SSL credentials. - if client_cert_source: - cert, key = client_cert_source() - ssl_credentials = grpc.ssl_channel_credentials( - certificate_chain=cert, private_key=key - ) - else: - ssl_credentials = SslCredentials().ssl_credentials - # create a new channel. The provided one is ignored. - self._grpc_channel = type(self).create_channel( - host, - credentials=credentials, - credentials_file=credentials_file, - ssl_credentials=ssl_credentials, - scopes=scopes or self.AUTH_SCOPES, - quota_project_id=quota_project_id, - options=[ - ("grpc.max_send_message_length", -1), - ("grpc.max_receive_message_length", -1), - ], - ) - self._ssl_channel_credentials = ssl_credentials else: - host = host if ":" in host else host + ":443" + if api_mtls_endpoint: + host = api_mtls_endpoint + + # Create SSL credentials with client_cert_source or application + # default SSL credentials. + if client_cert_source: + cert, key = client_cert_source() + self._ssl_channel_credentials = grpc.ssl_channel_credentials( + certificate_chain=cert, private_key=key + ) + else: + self._ssl_channel_credentials = SslCredentials().ssl_credentials - if credentials is None: - credentials, _ = auth.default( - scopes=self.AUTH_SCOPES, quota_project_id=quota_project_id - ) + else: + if client_cert_source_for_mtls and not ssl_channel_credentials: + cert, key = client_cert_source_for_mtls() + self._ssl_channel_credentials = grpc.ssl_channel_credentials( + certificate_chain=cert, private_key=key + ) - if client_cert_source_for_mtls and not ssl_channel_credentials: - cert, key = client_cert_source_for_mtls() - self._ssl_channel_credentials = grpc.ssl_channel_credentials( - certificate_chain=cert, private_key=key - ) + # The base transport sets the host, credentials and scopes + super().__init__( + host=host, + credentials=credentials, + credentials_file=credentials_file, + scopes=scopes, + quota_project_id=quota_project_id, + client_info=client_info, + ) - # create a new channel. The provided one is ignored. + if not self._grpc_channel: self._grpc_channel = type(self).create_channel( - host, - credentials=credentials, + self._host, + credentials=self._credentials, credentials_file=credentials_file, + scopes=self._scopes, ssl_credentials=self._ssl_channel_credentials, - scopes=scopes or self.AUTH_SCOPES, quota_project_id=quota_project_id, options=[ ("grpc.max_send_message_length", -1), @@ -187,17 +169,8 @@ def __init__( ], ) - self._stubs = {} # type: Dict[str, Callable] - - # Run the base constructor. - super().__init__( - host=host, - credentials=credentials, - credentials_file=credentials_file, - scopes=scopes or self.AUTH_SCOPES, - quota_project_id=quota_project_id, - client_info=client_info, - ) + # Wrap messages. This must be done after self._grpc_channel exists + self._prep_wrapped_messages(client_info) @classmethod def create_channel( @@ -211,7 +184,7 @@ def create_channel( ) -> grpc.Channel: """Create and return a gRPC channel object. Args: - address (Optional[str]): The host for the channel to use. + host (Optional[str]): The host for the channel to use. credentials (Optional[~.Credentials]): The authorization credentials to attach to requests. These credentials identify this application to the service. If diff --git a/google/cloud/bigquery_storage_v1/services/big_query_read/transports/grpc_asyncio.py b/google/cloud/bigquery_storage_v1/services/big_query_read/transports/grpc_asyncio.py index 919e6215..5af58e14 100644 --- a/google/cloud/bigquery_storage_v1/services/big_query_read/transports/grpc_asyncio.py +++ b/google/cloud/bigquery_storage_v1/services/big_query_read/transports/grpc_asyncio.py @@ -63,7 +63,7 @@ def create_channel( ) -> aio.Channel: """Create and return a gRPC AsyncIO channel object. Args: - address (Optional[str]): The host for the channel to use. + host (Optional[str]): The host for the channel to use. credentials (Optional[~.Credentials]): The authorization credentials to attach to requests. These credentials identify this application to the service. If @@ -141,10 +141,10 @@ def __init__( ignored if ``channel`` or ``ssl_channel_credentials`` is provided. quota_project_id (Optional[str]): An optional project to use for billing and quota. - client_info (google.api_core.gapic_v1.client_info.ClientInfo): - The client info used to send a user-agent string along with - API requests. If ``None``, then default info will be used. - Generally, you only need to set this if you're developing + client_info (google.api_core.gapic_v1.client_info.ClientInfo): + The client info used to send a user-agent string along with + API requests. If ``None``, then default info will be used. + Generally, you only need to set this if you're developing your own client library. Raises: @@ -153,7 +153,9 @@ def __init__( google.api_core.exceptions.DuplicateCredentialArgs: If both ``credentials`` and ``credentials_file`` are passed. """ + self._grpc_channel = None self._ssl_channel_credentials = ssl_channel_credentials + self._stubs: Dict[str, Callable] = {} if api_mtls_endpoint: warnings.warn("api_mtls_endpoint is deprecated", DeprecationWarning) @@ -161,70 +163,50 @@ def __init__( warnings.warn("client_cert_source is deprecated", DeprecationWarning) if channel: - # Sanity check: Ensure that channel and credentials are not both - # provided. + # Ignore credentials if a channel was passed. credentials = False - # If a channel was explicitly provided, set it. self._grpc_channel = channel self._ssl_channel_credentials = None - elif api_mtls_endpoint: - host = ( - api_mtls_endpoint - if ":" in api_mtls_endpoint - else api_mtls_endpoint + ":443" - ) - - if credentials is None: - credentials, _ = auth.default( - scopes=self.AUTH_SCOPES, quota_project_id=quota_project_id - ) - - # Create SSL credentials with client_cert_source or application - # default SSL credentials. - if client_cert_source: - cert, key = client_cert_source() - ssl_credentials = grpc.ssl_channel_credentials( - certificate_chain=cert, private_key=key - ) - else: - ssl_credentials = SslCredentials().ssl_credentials - # create a new channel. The provided one is ignored. - self._grpc_channel = type(self).create_channel( - host, - credentials=credentials, - credentials_file=credentials_file, - ssl_credentials=ssl_credentials, - scopes=scopes or self.AUTH_SCOPES, - quota_project_id=quota_project_id, - options=[ - ("grpc.max_send_message_length", -1), - ("grpc.max_receive_message_length", -1), - ], - ) - self._ssl_channel_credentials = ssl_credentials else: - host = host if ":" in host else host + ":443" + if api_mtls_endpoint: + host = api_mtls_endpoint + + # Create SSL credentials with client_cert_source or application + # default SSL credentials. + if client_cert_source: + cert, key = client_cert_source() + self._ssl_channel_credentials = grpc.ssl_channel_credentials( + certificate_chain=cert, private_key=key + ) + else: + self._ssl_channel_credentials = SslCredentials().ssl_credentials - if credentials is None: - credentials, _ = auth.default( - scopes=self.AUTH_SCOPES, quota_project_id=quota_project_id - ) + else: + if client_cert_source_for_mtls and not ssl_channel_credentials: + cert, key = client_cert_source_for_mtls() + self._ssl_channel_credentials = grpc.ssl_channel_credentials( + certificate_chain=cert, private_key=key + ) - if client_cert_source_for_mtls and not ssl_channel_credentials: - cert, key = client_cert_source_for_mtls() - self._ssl_channel_credentials = grpc.ssl_channel_credentials( - certificate_chain=cert, private_key=key - ) + # The base transport sets the host, credentials and scopes + super().__init__( + host=host, + credentials=credentials, + credentials_file=credentials_file, + scopes=scopes, + quota_project_id=quota_project_id, + client_info=client_info, + ) - # create a new channel. The provided one is ignored. + if not self._grpc_channel: self._grpc_channel = type(self).create_channel( - host, - credentials=credentials, + self._host, + credentials=self._credentials, credentials_file=credentials_file, + scopes=self._scopes, ssl_credentials=self._ssl_channel_credentials, - scopes=scopes or self.AUTH_SCOPES, quota_project_id=quota_project_id, options=[ ("grpc.max_send_message_length", -1), @@ -232,17 +214,8 @@ def __init__( ], ) - # Run the base constructor. - super().__init__( - host=host, - credentials=credentials, - credentials_file=credentials_file, - scopes=scopes or self.AUTH_SCOPES, - quota_project_id=quota_project_id, - client_info=client_info, - ) - - self._stubs = {} + # Wrap messages. This must be done after self._grpc_channel exists + self._prep_wrapped_messages(client_info) @property def grpc_channel(self) -> aio.Channel: diff --git a/google/cloud/bigquery_storage_v1beta2/services/big_query_read/async_client.py b/google/cloud/bigquery_storage_v1beta2/services/big_query_read/async_client.py index 47bee37d..7fc8f127 100644 --- a/google/cloud/bigquery_storage_v1beta2/services/big_query_read/async_client.py +++ b/google/cloud/bigquery_storage_v1beta2/services/big_query_read/async_client.py @@ -290,6 +290,7 @@ async def create_read_session( predicate=retries.if_exception_type( exceptions.DeadlineExceeded, exceptions.ServiceUnavailable, ), + deadline=600.0, ), default_timeout=600.0, client_info=DEFAULT_CLIENT_INFO, @@ -388,6 +389,7 @@ def read_rows( maximum=60.0, multiplier=1.3, predicate=retries.if_exception_type(exceptions.ServiceUnavailable,), + deadline=86400.0, ), default_timeout=86400.0, client_info=DEFAULT_CLIENT_INFO, @@ -459,6 +461,7 @@ async def split_read_stream( predicate=retries.if_exception_type( exceptions.DeadlineExceeded, exceptions.ServiceUnavailable, ), + deadline=600.0, ), default_timeout=600.0, client_info=DEFAULT_CLIENT_INFO, diff --git a/google/cloud/bigquery_storage_v1beta2/services/big_query_read/transports/base.py b/google/cloud/bigquery_storage_v1beta2/services/big_query_read/transports/base.py index 72f43ab7..5862d1f9 100644 --- a/google/cloud/bigquery_storage_v1beta2/services/big_query_read/transports/base.py +++ b/google/cloud/bigquery_storage_v1beta2/services/big_query_read/transports/base.py @@ -74,10 +74,10 @@ def __init__( scope (Optional[Sequence[str]]): A list of scopes. quota_project_id (Optional[str]): An optional project to use for billing and quota. - client_info (google.api_core.gapic_v1.client_info.ClientInfo): - The client info used to send a user-agent string along with - API requests. If ``None``, then default info will be used. - Generally, you only need to set this if you're developing + client_info (google.api_core.gapic_v1.client_info.ClientInfo): + The client info used to send a user-agent string along with + API requests. If ``None``, then default info will be used. + Generally, you only need to set this if you're developing your own client library. """ # Save the hostname. Default to port 443 (HTTPS) if none is specified. @@ -85,6 +85,9 @@ def __init__( host += ":443" self._host = host + # Save the scopes. + self._scopes = scopes or self.AUTH_SCOPES + # If no credentials are provided, then determine the appropriate # defaults. if credentials and credentials_file: @@ -94,20 +97,17 @@ def __init__( if credentials_file is not None: credentials, _ = auth.load_credentials_from_file( - credentials_file, scopes=scopes, quota_project_id=quota_project_id + credentials_file, scopes=self._scopes, quota_project_id=quota_project_id ) elif credentials is None: credentials, _ = auth.default( - scopes=scopes, quota_project_id=quota_project_id + scopes=self._scopes, quota_project_id=quota_project_id ) # Save the credentials. self._credentials = credentials - # Lifted into its own function so it can be stubbed out during tests. - self._prep_wrapped_messages(client_info) - def _prep_wrapped_messages(self, client_info): # Precompute the wrapped methods. self._wrapped_methods = { @@ -120,6 +120,7 @@ def _prep_wrapped_messages(self, client_info): predicate=retries.if_exception_type( exceptions.DeadlineExceeded, exceptions.ServiceUnavailable, ), + deadline=600.0, ), default_timeout=600.0, client_info=client_info, @@ -131,6 +132,7 @@ def _prep_wrapped_messages(self, client_info): maximum=60.0, multiplier=1.3, predicate=retries.if_exception_type(exceptions.ServiceUnavailable,), + deadline=86400.0, ), default_timeout=86400.0, client_info=client_info, @@ -144,6 +146,7 @@ def _prep_wrapped_messages(self, client_info): predicate=retries.if_exception_type( exceptions.DeadlineExceeded, exceptions.ServiceUnavailable, ), + deadline=600.0, ), default_timeout=600.0, client_info=client_info, diff --git a/google/cloud/bigquery_storage_v1beta2/services/big_query_read/transports/grpc.py b/google/cloud/bigquery_storage_v1beta2/services/big_query_read/transports/grpc.py index 2425d22f..dc2a433b 100644 --- a/google/cloud/bigquery_storage_v1beta2/services/big_query_read/transports/grpc.py +++ b/google/cloud/bigquery_storage_v1beta2/services/big_query_read/transports/grpc.py @@ -110,7 +110,9 @@ def __init__( google.api_core.exceptions.DuplicateCredentialArgs: If both ``credentials`` and ``credentials_file`` are passed. """ + self._grpc_channel = None self._ssl_channel_credentials = ssl_channel_credentials + self._stubs: Dict[str, Callable] = {} if api_mtls_endpoint: warnings.warn("api_mtls_endpoint is deprecated", DeprecationWarning) @@ -118,70 +120,50 @@ def __init__( warnings.warn("client_cert_source is deprecated", DeprecationWarning) if channel: - # Sanity check: Ensure that channel and credentials are not both - # provided. + # Ignore credentials if a channel was passed. credentials = False - # If a channel was explicitly provided, set it. self._grpc_channel = channel self._ssl_channel_credentials = None - elif api_mtls_endpoint: - host = ( - api_mtls_endpoint - if ":" in api_mtls_endpoint - else api_mtls_endpoint + ":443" - ) - - if credentials is None: - credentials, _ = auth.default( - scopes=self.AUTH_SCOPES, quota_project_id=quota_project_id - ) - - # Create SSL credentials with client_cert_source or application - # default SSL credentials. - if client_cert_source: - cert, key = client_cert_source() - ssl_credentials = grpc.ssl_channel_credentials( - certificate_chain=cert, private_key=key - ) - else: - ssl_credentials = SslCredentials().ssl_credentials - # create a new channel. The provided one is ignored. - self._grpc_channel = type(self).create_channel( - host, - credentials=credentials, - credentials_file=credentials_file, - ssl_credentials=ssl_credentials, - scopes=scopes or self.AUTH_SCOPES, - quota_project_id=quota_project_id, - options=[ - ("grpc.max_send_message_length", -1), - ("grpc.max_receive_message_length", -1), - ], - ) - self._ssl_channel_credentials = ssl_credentials else: - host = host if ":" in host else host + ":443" + if api_mtls_endpoint: + host = api_mtls_endpoint + + # Create SSL credentials with client_cert_source or application + # default SSL credentials. + if client_cert_source: + cert, key = client_cert_source() + self._ssl_channel_credentials = grpc.ssl_channel_credentials( + certificate_chain=cert, private_key=key + ) + else: + self._ssl_channel_credentials = SslCredentials().ssl_credentials - if credentials is None: - credentials, _ = auth.default( - scopes=self.AUTH_SCOPES, quota_project_id=quota_project_id - ) + else: + if client_cert_source_for_mtls and not ssl_channel_credentials: + cert, key = client_cert_source_for_mtls() + self._ssl_channel_credentials = grpc.ssl_channel_credentials( + certificate_chain=cert, private_key=key + ) - if client_cert_source_for_mtls and not ssl_channel_credentials: - cert, key = client_cert_source_for_mtls() - self._ssl_channel_credentials = grpc.ssl_channel_credentials( - certificate_chain=cert, private_key=key - ) + # The base transport sets the host, credentials and scopes + super().__init__( + host=host, + credentials=credentials, + credentials_file=credentials_file, + scopes=scopes, + quota_project_id=quota_project_id, + client_info=client_info, + ) - # create a new channel. The provided one is ignored. + if not self._grpc_channel: self._grpc_channel = type(self).create_channel( - host, - credentials=credentials, + self._host, + credentials=self._credentials, credentials_file=credentials_file, + scopes=self._scopes, ssl_credentials=self._ssl_channel_credentials, - scopes=scopes or self.AUTH_SCOPES, quota_project_id=quota_project_id, options=[ ("grpc.max_send_message_length", -1), @@ -189,17 +171,8 @@ def __init__( ], ) - self._stubs = {} # type: Dict[str, Callable] - - # Run the base constructor. - super().__init__( - host=host, - credentials=credentials, - credentials_file=credentials_file, - scopes=scopes or self.AUTH_SCOPES, - quota_project_id=quota_project_id, - client_info=client_info, - ) + # Wrap messages. This must be done after self._grpc_channel exists + self._prep_wrapped_messages(client_info) @classmethod def create_channel( @@ -213,7 +186,7 @@ def create_channel( ) -> grpc.Channel: """Create and return a gRPC channel object. Args: - address (Optional[str]): The host for the channel to use. + host (Optional[str]): The host for the channel to use. credentials (Optional[~.Credentials]): The authorization credentials to attach to requests. These credentials identify this application to the service. If diff --git a/google/cloud/bigquery_storage_v1beta2/services/big_query_read/transports/grpc_asyncio.py b/google/cloud/bigquery_storage_v1beta2/services/big_query_read/transports/grpc_asyncio.py index f8021d05..0691a5aa 100644 --- a/google/cloud/bigquery_storage_v1beta2/services/big_query_read/transports/grpc_asyncio.py +++ b/google/cloud/bigquery_storage_v1beta2/services/big_query_read/transports/grpc_asyncio.py @@ -65,7 +65,7 @@ def create_channel( ) -> aio.Channel: """Create and return a gRPC AsyncIO channel object. Args: - address (Optional[str]): The host for the channel to use. + host (Optional[str]): The host for the channel to use. credentials (Optional[~.Credentials]): The authorization credentials to attach to requests. These credentials identify this application to the service. If @@ -143,10 +143,10 @@ def __init__( ignored if ``channel`` or ``ssl_channel_credentials`` is provided. quota_project_id (Optional[str]): An optional project to use for billing and quota. - client_info (google.api_core.gapic_v1.client_info.ClientInfo): - The client info used to send a user-agent string along with - API requests. If ``None``, then default info will be used. - Generally, you only need to set this if you're developing + client_info (google.api_core.gapic_v1.client_info.ClientInfo): + The client info used to send a user-agent string along with + API requests. If ``None``, then default info will be used. + Generally, you only need to set this if you're developing your own client library. Raises: @@ -155,7 +155,9 @@ def __init__( google.api_core.exceptions.DuplicateCredentialArgs: If both ``credentials`` and ``credentials_file`` are passed. """ + self._grpc_channel = None self._ssl_channel_credentials = ssl_channel_credentials + self._stubs: Dict[str, Callable] = {} if api_mtls_endpoint: warnings.warn("api_mtls_endpoint is deprecated", DeprecationWarning) @@ -163,70 +165,50 @@ def __init__( warnings.warn("client_cert_source is deprecated", DeprecationWarning) if channel: - # Sanity check: Ensure that channel and credentials are not both - # provided. + # Ignore credentials if a channel was passed. credentials = False - # If a channel was explicitly provided, set it. self._grpc_channel = channel self._ssl_channel_credentials = None - elif api_mtls_endpoint: - host = ( - api_mtls_endpoint - if ":" in api_mtls_endpoint - else api_mtls_endpoint + ":443" - ) - - if credentials is None: - credentials, _ = auth.default( - scopes=self.AUTH_SCOPES, quota_project_id=quota_project_id - ) - - # Create SSL credentials with client_cert_source or application - # default SSL credentials. - if client_cert_source: - cert, key = client_cert_source() - ssl_credentials = grpc.ssl_channel_credentials( - certificate_chain=cert, private_key=key - ) - else: - ssl_credentials = SslCredentials().ssl_credentials - # create a new channel. The provided one is ignored. - self._grpc_channel = type(self).create_channel( - host, - credentials=credentials, - credentials_file=credentials_file, - ssl_credentials=ssl_credentials, - scopes=scopes or self.AUTH_SCOPES, - quota_project_id=quota_project_id, - options=[ - ("grpc.max_send_message_length", -1), - ("grpc.max_receive_message_length", -1), - ], - ) - self._ssl_channel_credentials = ssl_credentials else: - host = host if ":" in host else host + ":443" + if api_mtls_endpoint: + host = api_mtls_endpoint + + # Create SSL credentials with client_cert_source or application + # default SSL credentials. + if client_cert_source: + cert, key = client_cert_source() + self._ssl_channel_credentials = grpc.ssl_channel_credentials( + certificate_chain=cert, private_key=key + ) + else: + self._ssl_channel_credentials = SslCredentials().ssl_credentials - if credentials is None: - credentials, _ = auth.default( - scopes=self.AUTH_SCOPES, quota_project_id=quota_project_id - ) + else: + if client_cert_source_for_mtls and not ssl_channel_credentials: + cert, key = client_cert_source_for_mtls() + self._ssl_channel_credentials = grpc.ssl_channel_credentials( + certificate_chain=cert, private_key=key + ) - if client_cert_source_for_mtls and not ssl_channel_credentials: - cert, key = client_cert_source_for_mtls() - self._ssl_channel_credentials = grpc.ssl_channel_credentials( - certificate_chain=cert, private_key=key - ) + # The base transport sets the host, credentials and scopes + super().__init__( + host=host, + credentials=credentials, + credentials_file=credentials_file, + scopes=scopes, + quota_project_id=quota_project_id, + client_info=client_info, + ) - # create a new channel. The provided one is ignored. + if not self._grpc_channel: self._grpc_channel = type(self).create_channel( - host, - credentials=credentials, + self._host, + credentials=self._credentials, credentials_file=credentials_file, + scopes=self._scopes, ssl_credentials=self._ssl_channel_credentials, - scopes=scopes or self.AUTH_SCOPES, quota_project_id=quota_project_id, options=[ ("grpc.max_send_message_length", -1), @@ -234,17 +216,8 @@ def __init__( ], ) - # Run the base constructor. - super().__init__( - host=host, - credentials=credentials, - credentials_file=credentials_file, - scopes=scopes or self.AUTH_SCOPES, - quota_project_id=quota_project_id, - client_info=client_info, - ) - - self._stubs = {} + # Wrap messages. This must be done after self._grpc_channel exists + self._prep_wrapped_messages(client_info) @property def grpc_channel(self) -> aio.Channel: diff --git a/google/cloud/bigquery_storage_v1beta2/services/big_query_write/async_client.py b/google/cloud/bigquery_storage_v1beta2/services/big_query_write/async_client.py index b3793a40..ab60a9a7 100644 --- a/google/cloud/bigquery_storage_v1beta2/services/big_query_write/async_client.py +++ b/google/cloud/bigquery_storage_v1beta2/services/big_query_write/async_client.py @@ -267,6 +267,7 @@ async def create_write_stream( exceptions.ResourceExhausted, exceptions.ServiceUnavailable, ), + deadline=600.0, ), default_timeout=600.0, client_info=DEFAULT_CLIENT_INFO, @@ -339,6 +340,7 @@ def append_rows( predicate=retries.if_exception_type( exceptions.ResourceExhausted, exceptions.ServiceUnavailable, ), + deadline=86400.0, ), default_timeout=86400.0, client_info=DEFAULT_CLIENT_INFO, @@ -419,6 +421,7 @@ async def get_write_stream( predicate=retries.if_exception_type( exceptions.DeadlineExceeded, exceptions.ServiceUnavailable, ), + deadline=600.0, ), default_timeout=600.0, client_info=DEFAULT_CLIENT_INFO, @@ -499,6 +502,7 @@ async def finalize_write_stream( predicate=retries.if_exception_type( exceptions.DeadlineExceeded, exceptions.ServiceUnavailable, ), + deadline=600.0, ), default_timeout=600.0, client_info=DEFAULT_CLIENT_INFO, @@ -583,6 +587,7 @@ async def batch_commit_write_streams( predicate=retries.if_exception_type( exceptions.DeadlineExceeded, exceptions.ServiceUnavailable, ), + deadline=600.0, ), default_timeout=600.0, client_info=DEFAULT_CLIENT_INFO, @@ -666,6 +671,7 @@ async def flush_rows( predicate=retries.if_exception_type( exceptions.DeadlineExceeded, exceptions.ServiceUnavailable, ), + deadline=600.0, ), default_timeout=600.0, client_info=DEFAULT_CLIENT_INFO, diff --git a/google/cloud/bigquery_storage_v1beta2/services/big_query_write/transports/base.py b/google/cloud/bigquery_storage_v1beta2/services/big_query_write/transports/base.py index 6b9b6a18..4682188d 100644 --- a/google/cloud/bigquery_storage_v1beta2/services/big_query_write/transports/base.py +++ b/google/cloud/bigquery_storage_v1beta2/services/big_query_write/transports/base.py @@ -74,10 +74,10 @@ def __init__( scope (Optional[Sequence[str]]): A list of scopes. quota_project_id (Optional[str]): An optional project to use for billing and quota. - client_info (google.api_core.gapic_v1.client_info.ClientInfo): - The client info used to send a user-agent string along with - API requests. If ``None``, then default info will be used. - Generally, you only need to set this if you're developing + client_info (google.api_core.gapic_v1.client_info.ClientInfo): + The client info used to send a user-agent string along with + API requests. If ``None``, then default info will be used. + Generally, you only need to set this if you're developing your own client library. """ # Save the hostname. Default to port 443 (HTTPS) if none is specified. @@ -85,6 +85,9 @@ def __init__( host += ":443" self._host = host + # Save the scopes. + self._scopes = scopes or self.AUTH_SCOPES + # If no credentials are provided, then determine the appropriate # defaults. if credentials and credentials_file: @@ -94,20 +97,17 @@ def __init__( if credentials_file is not None: credentials, _ = auth.load_credentials_from_file( - credentials_file, scopes=scopes, quota_project_id=quota_project_id + credentials_file, scopes=self._scopes, quota_project_id=quota_project_id ) elif credentials is None: credentials, _ = auth.default( - scopes=scopes, quota_project_id=quota_project_id + scopes=self._scopes, quota_project_id=quota_project_id ) # Save the credentials. self._credentials = credentials - # Lifted into its own function so it can be stubbed out during tests. - self._prep_wrapped_messages(client_info) - def _prep_wrapped_messages(self, client_info): # Precompute the wrapped methods. self._wrapped_methods = { @@ -122,6 +122,7 @@ def _prep_wrapped_messages(self, client_info): exceptions.ResourceExhausted, exceptions.ServiceUnavailable, ), + deadline=600.0, ), default_timeout=600.0, client_info=client_info, @@ -135,6 +136,7 @@ def _prep_wrapped_messages(self, client_info): predicate=retries.if_exception_type( exceptions.ResourceExhausted, exceptions.ServiceUnavailable, ), + deadline=86400.0, ), default_timeout=86400.0, client_info=client_info, @@ -148,6 +150,7 @@ def _prep_wrapped_messages(self, client_info): predicate=retries.if_exception_type( exceptions.DeadlineExceeded, exceptions.ServiceUnavailable, ), + deadline=600.0, ), default_timeout=600.0, client_info=client_info, @@ -161,6 +164,7 @@ def _prep_wrapped_messages(self, client_info): predicate=retries.if_exception_type( exceptions.DeadlineExceeded, exceptions.ServiceUnavailable, ), + deadline=600.0, ), default_timeout=600.0, client_info=client_info, @@ -174,6 +178,7 @@ def _prep_wrapped_messages(self, client_info): predicate=retries.if_exception_type( exceptions.DeadlineExceeded, exceptions.ServiceUnavailable, ), + deadline=600.0, ), default_timeout=600.0, client_info=client_info, @@ -187,6 +192,7 @@ def _prep_wrapped_messages(self, client_info): predicate=retries.if_exception_type( exceptions.DeadlineExceeded, exceptions.ServiceUnavailable, ), + deadline=600.0, ), default_timeout=600.0, client_info=client_info, diff --git a/google/cloud/bigquery_storage_v1beta2/services/big_query_write/transports/grpc.py b/google/cloud/bigquery_storage_v1beta2/services/big_query_write/transports/grpc.py index 626c960a..2fd1cebd 100644 --- a/google/cloud/bigquery_storage_v1beta2/services/big_query_write/transports/grpc.py +++ b/google/cloud/bigquery_storage_v1beta2/services/big_query_write/transports/grpc.py @@ -108,7 +108,9 @@ def __init__( google.api_core.exceptions.DuplicateCredentialArgs: If both ``credentials`` and ``credentials_file`` are passed. """ + self._grpc_channel = None self._ssl_channel_credentials = ssl_channel_credentials + self._stubs: Dict[str, Callable] = {} if api_mtls_endpoint: warnings.warn("api_mtls_endpoint is deprecated", DeprecationWarning) @@ -116,70 +118,50 @@ def __init__( warnings.warn("client_cert_source is deprecated", DeprecationWarning) if channel: - # Sanity check: Ensure that channel and credentials are not both - # provided. + # Ignore credentials if a channel was passed. credentials = False - # If a channel was explicitly provided, set it. self._grpc_channel = channel self._ssl_channel_credentials = None - elif api_mtls_endpoint: - host = ( - api_mtls_endpoint - if ":" in api_mtls_endpoint - else api_mtls_endpoint + ":443" - ) - - if credentials is None: - credentials, _ = auth.default( - scopes=self.AUTH_SCOPES, quota_project_id=quota_project_id - ) - - # Create SSL credentials with client_cert_source or application - # default SSL credentials. - if client_cert_source: - cert, key = client_cert_source() - ssl_credentials = grpc.ssl_channel_credentials( - certificate_chain=cert, private_key=key - ) - else: - ssl_credentials = SslCredentials().ssl_credentials - # create a new channel. The provided one is ignored. - self._grpc_channel = type(self).create_channel( - host, - credentials=credentials, - credentials_file=credentials_file, - ssl_credentials=ssl_credentials, - scopes=scopes or self.AUTH_SCOPES, - quota_project_id=quota_project_id, - options=[ - ("grpc.max_send_message_length", -1), - ("grpc.max_receive_message_length", -1), - ], - ) - self._ssl_channel_credentials = ssl_credentials else: - host = host if ":" in host else host + ":443" + if api_mtls_endpoint: + host = api_mtls_endpoint + + # Create SSL credentials with client_cert_source or application + # default SSL credentials. + if client_cert_source: + cert, key = client_cert_source() + self._ssl_channel_credentials = grpc.ssl_channel_credentials( + certificate_chain=cert, private_key=key + ) + else: + self._ssl_channel_credentials = SslCredentials().ssl_credentials - if credentials is None: - credentials, _ = auth.default( - scopes=self.AUTH_SCOPES, quota_project_id=quota_project_id - ) + else: + if client_cert_source_for_mtls and not ssl_channel_credentials: + cert, key = client_cert_source_for_mtls() + self._ssl_channel_credentials = grpc.ssl_channel_credentials( + certificate_chain=cert, private_key=key + ) - if client_cert_source_for_mtls and not ssl_channel_credentials: - cert, key = client_cert_source_for_mtls() - self._ssl_channel_credentials = grpc.ssl_channel_credentials( - certificate_chain=cert, private_key=key - ) + # The base transport sets the host, credentials and scopes + super().__init__( + host=host, + credentials=credentials, + credentials_file=credentials_file, + scopes=scopes, + quota_project_id=quota_project_id, + client_info=client_info, + ) - # create a new channel. The provided one is ignored. + if not self._grpc_channel: self._grpc_channel = type(self).create_channel( - host, - credentials=credentials, + self._host, + credentials=self._credentials, credentials_file=credentials_file, + scopes=self._scopes, ssl_credentials=self._ssl_channel_credentials, - scopes=scopes or self.AUTH_SCOPES, quota_project_id=quota_project_id, options=[ ("grpc.max_send_message_length", -1), @@ -187,17 +169,8 @@ def __init__( ], ) - self._stubs = {} # type: Dict[str, Callable] - - # Run the base constructor. - super().__init__( - host=host, - credentials=credentials, - credentials_file=credentials_file, - scopes=scopes or self.AUTH_SCOPES, - quota_project_id=quota_project_id, - client_info=client_info, - ) + # Wrap messages. This must be done after self._grpc_channel exists + self._prep_wrapped_messages(client_info) @classmethod def create_channel( @@ -211,7 +184,7 @@ def create_channel( ) -> grpc.Channel: """Create and return a gRPC channel object. Args: - address (Optional[str]): The host for the channel to use. + host (Optional[str]): The host for the channel to use. credentials (Optional[~.Credentials]): The authorization credentials to attach to requests. These credentials identify this application to the service. If diff --git a/google/cloud/bigquery_storage_v1beta2/services/big_query_write/transports/grpc_asyncio.py b/google/cloud/bigquery_storage_v1beta2/services/big_query_write/transports/grpc_asyncio.py index a053b100..890f7572 100644 --- a/google/cloud/bigquery_storage_v1beta2/services/big_query_write/transports/grpc_asyncio.py +++ b/google/cloud/bigquery_storage_v1beta2/services/big_query_write/transports/grpc_asyncio.py @@ -63,7 +63,7 @@ def create_channel( ) -> aio.Channel: """Create and return a gRPC AsyncIO channel object. Args: - address (Optional[str]): The host for the channel to use. + host (Optional[str]): The host for the channel to use. credentials (Optional[~.Credentials]): The authorization credentials to attach to requests. These credentials identify this application to the service. If @@ -141,10 +141,10 @@ def __init__( ignored if ``channel`` or ``ssl_channel_credentials`` is provided. quota_project_id (Optional[str]): An optional project to use for billing and quota. - client_info (google.api_core.gapic_v1.client_info.ClientInfo): - The client info used to send a user-agent string along with - API requests. If ``None``, then default info will be used. - Generally, you only need to set this if you're developing + client_info (google.api_core.gapic_v1.client_info.ClientInfo): + The client info used to send a user-agent string along with + API requests. If ``None``, then default info will be used. + Generally, you only need to set this if you're developing your own client library. Raises: @@ -153,7 +153,9 @@ def __init__( google.api_core.exceptions.DuplicateCredentialArgs: If both ``credentials`` and ``credentials_file`` are passed. """ + self._grpc_channel = None self._ssl_channel_credentials = ssl_channel_credentials + self._stubs: Dict[str, Callable] = {} if api_mtls_endpoint: warnings.warn("api_mtls_endpoint is deprecated", DeprecationWarning) @@ -161,70 +163,50 @@ def __init__( warnings.warn("client_cert_source is deprecated", DeprecationWarning) if channel: - # Sanity check: Ensure that channel and credentials are not both - # provided. + # Ignore credentials if a channel was passed. credentials = False - # If a channel was explicitly provided, set it. self._grpc_channel = channel self._ssl_channel_credentials = None - elif api_mtls_endpoint: - host = ( - api_mtls_endpoint - if ":" in api_mtls_endpoint - else api_mtls_endpoint + ":443" - ) - - if credentials is None: - credentials, _ = auth.default( - scopes=self.AUTH_SCOPES, quota_project_id=quota_project_id - ) - - # Create SSL credentials with client_cert_source or application - # default SSL credentials. - if client_cert_source: - cert, key = client_cert_source() - ssl_credentials = grpc.ssl_channel_credentials( - certificate_chain=cert, private_key=key - ) - else: - ssl_credentials = SslCredentials().ssl_credentials - # create a new channel. The provided one is ignored. - self._grpc_channel = type(self).create_channel( - host, - credentials=credentials, - credentials_file=credentials_file, - ssl_credentials=ssl_credentials, - scopes=scopes or self.AUTH_SCOPES, - quota_project_id=quota_project_id, - options=[ - ("grpc.max_send_message_length", -1), - ("grpc.max_receive_message_length", -1), - ], - ) - self._ssl_channel_credentials = ssl_credentials else: - host = host if ":" in host else host + ":443" + if api_mtls_endpoint: + host = api_mtls_endpoint + + # Create SSL credentials with client_cert_source or application + # default SSL credentials. + if client_cert_source: + cert, key = client_cert_source() + self._ssl_channel_credentials = grpc.ssl_channel_credentials( + certificate_chain=cert, private_key=key + ) + else: + self._ssl_channel_credentials = SslCredentials().ssl_credentials - if credentials is None: - credentials, _ = auth.default( - scopes=self.AUTH_SCOPES, quota_project_id=quota_project_id - ) + else: + if client_cert_source_for_mtls and not ssl_channel_credentials: + cert, key = client_cert_source_for_mtls() + self._ssl_channel_credentials = grpc.ssl_channel_credentials( + certificate_chain=cert, private_key=key + ) - if client_cert_source_for_mtls and not ssl_channel_credentials: - cert, key = client_cert_source_for_mtls() - self._ssl_channel_credentials = grpc.ssl_channel_credentials( - certificate_chain=cert, private_key=key - ) + # The base transport sets the host, credentials and scopes + super().__init__( + host=host, + credentials=credentials, + credentials_file=credentials_file, + scopes=scopes, + quota_project_id=quota_project_id, + client_info=client_info, + ) - # create a new channel. The provided one is ignored. + if not self._grpc_channel: self._grpc_channel = type(self).create_channel( - host, - credentials=credentials, + self._host, + credentials=self._credentials, credentials_file=credentials_file, + scopes=self._scopes, ssl_credentials=self._ssl_channel_credentials, - scopes=scopes or self.AUTH_SCOPES, quota_project_id=quota_project_id, options=[ ("grpc.max_send_message_length", -1), @@ -232,17 +214,8 @@ def __init__( ], ) - # Run the base constructor. - super().__init__( - host=host, - credentials=credentials, - credentials_file=credentials_file, - scopes=scopes or self.AUTH_SCOPES, - quota_project_id=quota_project_id, - client_info=client_info, - ) - - self._stubs = {} + # Wrap messages. This must be done after self._grpc_channel exists + self._prep_wrapped_messages(client_info) @property def grpc_channel(self) -> aio.Channel: diff --git a/synth.metadata b/synth.metadata index f30677e6..e403759e 100644 --- a/synth.metadata +++ b/synth.metadata @@ -4,15 +4,15 @@ "git": { "name": ".", "remote": "https://github.com/googleapis/python-bigquery-storage.git", - "sha": "5a235c5cd194f8ca2fa7591032ba2cafd50a7dbc" + "sha": "2d981460e62960f273ceb2537a70aeda8cbde5ad" } }, { "git": { "name": "googleapis", "remote": "https://github.com/googleapis/googleapis.git", - "sha": "3aeb3a70f66457a9e6b07caff841719bb9873b57", - "internalRef": "361676678" + "sha": "149a3a84c29c9b8189576c7442ccb6dcf6a8f95b", + "internalRef": "364411656" } }, { From 8a977633a81d080f03f6922752adbf4284199dd4 Mon Sep 17 00:00:00 2001 From: Yoshi Automation Bot Date: Tue, 30 Mar 2021 15:03:47 -0700 Subject: [PATCH 18/24] deps: update minimum pandas to 0.21.1 (#165) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(deps): update precommit hook pycqa/flake8 to v3.9.0 [![WhiteSource Renovate](https://app.renovatebot.com/images/banner.svg)](https://renovatebot.com) This PR contains the following updates: | Package | Type | Update | Change | |---|---|---|---| | [pycqa/flake8](https://gitlab.com/pycqa/flake8) | repository | minor | `3.8.4` -> `3.9.0` | --- ### Release Notes
pycqa/flake8 ### [`v3.9.0`](https://gitlab.com/pycqa/flake8/compare/3.8.4...3.9.0) [Compare Source](https://gitlab.com/pycqa/flake8/compare/3.8.4...3.9.0)
--- ### Renovate configuration :date: **Schedule**: At any time (no schedule defined). :vertical_traffic_light: **Automerge**: Disabled by config. Please merge this manually once you are satisfied. :recycle: **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. :no_bell: **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR has been generated by [WhiteSource Renovate](https://renovate.whitesourcesoftware.com). View repository job log [here](https://app.renovatebot.com/dashboard#github/googleapis/synthtool). Source-Author: WhiteSource Renovate Source-Date: Tue Mar 23 17:38:03 2021 +0100 Source-Repo: googleapis/synthtool Source-Sha: f5c5904fb0c6aa3b3730eadf4e5a4485afc65726 Source-Link: https://github.com/googleapis/synthtool/commit/f5c5904fb0c6aa3b3730eadf4e5a4485afc65726 * test(python): use constraints files to check dependency lower bounds Use a constraints file when installing dependencies for system and unit tests nox sessions. https://pip.pypa.io/en/stable/user_guide/#constraints-files > Constraints files are requirements files that **only control which version of a requirement is installed, not whether it is installed or not**. Their syntax and contents is nearly identical to Requirements Files. There is one key difference: Including a package in a constraints file does not trigger installation of the package. ``` testing ├── constraints-3.10.txt ├── constraints-3.11.txt ├── constraints-3.6.txt ├── constraints-3.7.txt ├── constraints-3.8.txt └── constraints-3.9.txt ``` Going forward, one constraints file (currently 3.6) will be populated with every library requirement and extra listed in the `setup.py`. The constraints file will pin each requirement to the lower bound. This ensures that library maintainers will see test failures if they forget to update a lower bound on a dependency. See https://github.com/googleapis/python-bigquery/pull/263 for an example Source-Author: Bu Sun Kim <8822365+busunkim96@users.noreply.github.com> Source-Date: Tue Mar 23 10:52:02 2021 -0600 Source-Repo: googleapis/synthtool Source-Sha: 86ed43d4f56e6404d068e62e497029018879c771 Source-Link: https://github.com/googleapis/synthtool/commit/86ed43d4f56e6404d068e62e497029018879c771 * bump minimum pandas * update minimum pandas * use recommended way to localize timestamps in tests Co-authored-by: Tim Swast --- .pre-commit-config.yaml | 2 +- noxfile.py | 27 +++++++++++++++++++++------ setup.py | 2 +- synth.metadata | 6 +++--- testing/constraints-3.6.txt | 4 ++-- tests/unit/test_reader_v1.py | 4 +++- tests/unit/test_reader_v1_arrow.py | 4 +++- 7 files changed, 34 insertions(+), 15 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a9024b15..32302e48 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -12,6 +12,6 @@ repos: hooks: - id: black - repo: https://gitlab.com/pycqa/flake8 - rev: 3.8.4 + rev: 3.9.0 hooks: - id: flake8 diff --git a/noxfile.py b/noxfile.py index 0647537d..73ed1ac2 100644 --- a/noxfile.py +++ b/noxfile.py @@ -18,6 +18,7 @@ from __future__ import absolute_import import os +import pathlib import shutil import nox @@ -30,6 +31,8 @@ SYSTEM_TEST_PYTHON_VERSIONS = ["3.8"] UNIT_TEST_PYTHON_VERSIONS = ["3.6", "3.7", "3.8", "3.9"] +CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute() + # 'docfx' is excluded since it only needs to run in 'docs-presubmit' nox.options.sessions = [ "unit", @@ -84,13 +87,15 @@ def lint_setup_py(session): def default(session): # Install all test dependencies, then install this package in-place. - session.install("asyncmock", "pytest-asyncio") - session.install( - "mock", "pytest", "pytest-cov", + constraints_path = str( + CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt" ) + session.install("asyncmock", "pytest-asyncio", "-c", constraints_path) + + session.install("mock", "pytest", "pytest-cov", "-c", constraints_path) - session.install("-e", ".[fastavro,pandas,pyarrow]") + session.install("-e", ".[fastavro,pandas,pyarrow]", "-c", constraints_path) # Run py.test against the unit tests. session.run( @@ -117,6 +122,9 @@ def unit(session): @nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS) def system(session): """Run the system test suite.""" + constraints_path = str( + CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt" + ) system_test_path = os.path.join("tests", "system.py") system_test_folder_path = os.path.join("tests", "system") @@ -141,8 +149,15 @@ def system(session): # Install all test dependencies, then install this package into the # virtualenv's dist-packages. - session.install("mock", "pytest", "google-cloud-testutils", "google-cloud-bigquery") - session.install("-e", ".[fastavro,pandas,pyarrow]") + session.install( + "mock", + "pytest", + "google-cloud-testutils", + "google-cloud-bigquery", + "-c", + constraints_path, + ) + session.install("-e", ".[fastavro,pandas,pyarrow]", "-c", constraints_path) # Run py.test against the system tests. if system_test_exists: diff --git a/setup.py b/setup.py index 328177ce..e1896bd1 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ "libcst >= 0.2.5", ] extras = { - "pandas": "pandas>=0.17.1", + "pandas": "pandas>=0.21.1", "fastavro": "fastavro>=0.21.2", "pyarrow": "pyarrow>=0.15.0", } diff --git a/synth.metadata b/synth.metadata index e403759e..a2bc0bfc 100644 --- a/synth.metadata +++ b/synth.metadata @@ -4,7 +4,7 @@ "git": { "name": ".", "remote": "https://github.com/googleapis/python-bigquery-storage.git", - "sha": "2d981460e62960f273ceb2537a70aeda8cbde5ad" + "sha": "8c1071337ea6c76a61f601b46254e6f07d8f5add" } }, { @@ -19,14 +19,14 @@ "git": { "name": "synthtool", "remote": "https://github.com/googleapis/synthtool.git", - "sha": "79c8dd7ee768292f933012d3a69a5b4676404cda" + "sha": "86ed43d4f56e6404d068e62e497029018879c771" } }, { "git": { "name": "synthtool", "remote": "https://github.com/googleapis/synthtool.git", - "sha": "79c8dd7ee768292f933012d3a69a5b4676404cda" + "sha": "86ed43d4f56e6404d068e62e497029018879c771" } } ], diff --git a/testing/constraints-3.6.txt b/testing/constraints-3.6.txt index 60b1b798..f9a2ecfa 100644 --- a/testing/constraints-3.6.txt +++ b/testing/constraints-3.6.txt @@ -9,5 +9,5 @@ google-api-core==1.22.2 proto-plus==1.4.0 libcst==0.2.5 fastavro==0.21.2 -pandas==0.17.1 -pyarrow==0.15.0 \ No newline at end of file +pandas==0.21.1 +pyarrow==0.15.0 diff --git a/tests/unit/test_reader_v1.py b/tests/unit/test_reader_v1.py index fcefbca7..7fb8d5a4 100644 --- a/tests/unit/test_reader_v1.py +++ b/tests/unit/test_reader_v1.py @@ -427,7 +427,9 @@ def test_to_dataframe_empty_w_scalars_avro(class_under_test): expected["int_col"] = expected["int_col"].astype("int64") expected["float_col"] = expected["float_col"].astype("float64") expected["bool_col"] = expected["bool_col"].astype("bool") - expected["ts_col"] = expected["ts_col"].astype("datetime64[ns, UTC]") + expected["ts_col"] = ( + expected["ts_col"].astype("datetime64[ns]").dt.tz_localize("UTC") + ) pandas.testing.assert_frame_equal( got.reset_index(drop=True), # reset_index to ignore row labels diff --git a/tests/unit/test_reader_v1_arrow.py b/tests/unit/test_reader_v1_arrow.py index 202e0d81..492098f5 100644 --- a/tests/unit/test_reader_v1_arrow.py +++ b/tests/unit/test_reader_v1_arrow.py @@ -249,7 +249,9 @@ def test_to_dataframe_empty_w_scalars_arrow(class_under_test): expected["int_col"] = expected["int_col"].astype("int64") expected["float_col"] = expected["float_col"].astype("float64") expected["bool_col"] = expected["bool_col"].astype("bool") - expected["ts_col"] = expected["ts_col"].astype("datetime64[ns, UTC]") + expected["ts_col"] = ( + expected["ts_col"].astype("datetime64[ns]").dt.tz_localize("UTC") + ) pandas.testing.assert_frame_equal( got.reset_index(drop=True), # reset_index to ignore row labels From 1c91a276289a0e319f93b136836f81ee943f661c Mon Sep 17 00:00:00 2001 From: Yoshi Automation Bot Date: Tue, 30 Mar 2021 15:10:02 -0700 Subject: [PATCH 19/24] feat: add a Arrow compression options (Only LZ4 for now) (#166) Also: * feat: Return schema on first ReadRowsResponse. * doc: clarify limit on filter string. This PR was generated using Autosynth. :rainbow: Synth log will be available here: https://source.cloud.google.com/results/invocations/72a2a14b-0135-4939-ae4b-93b118a2b3e8/targets - [ ] To automatically regenerate this PR, check this box. (May take up to 24 hours.) PiperOrigin-RevId: 365759522 Source-Link: https://github.com/googleapis/googleapis/commit/c539b9b08b3366ee00c0ec1950f4df711552a269 --- google/cloud/bigquery_storage/__init__.py | 2 ++ .../bigquery_storage_v1/proto/arrow.proto | 19 +++++++++++-- .../bigquery_storage_v1/proto/avro.proto | 3 +- .../bigquery_storage_v1/proto/storage.proto | 22 ++++++++++++--- .../bigquery_storage_v1/proto/stream.proto | 28 ++++++++++++------- .../bigquery_storage_v1/types/__init__.py | 2 ++ .../cloud/bigquery_storage_v1/types/arrow.py | 19 ++++++++++++- .../bigquery_storage_v1/types/storage.py | 12 ++++++++ .../cloud/bigquery_storage_v1/types/stream.py | 13 ++++++++- synth.metadata | 4 +-- 10 files changed, 102 insertions(+), 22 deletions(-) diff --git a/google/cloud/bigquery_storage/__init__.py b/google/cloud/bigquery_storage/__init__.py index 227e6184..003bc480 100644 --- a/google/cloud/bigquery_storage/__init__.py +++ b/google/cloud/bigquery_storage/__init__.py @@ -20,6 +20,7 @@ from google.cloud.bigquery_storage_v1 import __version__ from google.cloud.bigquery_storage_v1.types.arrow import ArrowRecordBatch from google.cloud.bigquery_storage_v1.types.arrow import ArrowSchema +from google.cloud.bigquery_storage_v1.types.arrow import ArrowSerializationOptions from google.cloud.bigquery_storage_v1.types.avro import AvroRows from google.cloud.bigquery_storage_v1.types.avro import AvroSchema from google.cloud.bigquery_storage_v1.types.storage import CreateReadSessionRequest @@ -38,6 +39,7 @@ "types", "ArrowRecordBatch", "ArrowSchema", + "ArrowSerializationOptions", "AvroRows", "AvroSchema", "BigQueryReadClient", diff --git a/google/cloud/bigquery_storage_v1/proto/arrow.proto b/google/cloud/bigquery_storage_v1/proto/arrow.proto index 1c54eeab..4b240f52 100644 --- a/google/cloud/bigquery_storage_v1/proto/arrow.proto +++ b/google/cloud/bigquery_storage_v1/proto/arrow.proto @@ -1,4 +1,4 @@ -// Copyright 2019 Google LLC. +// Copyright 2021 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -11,7 +11,6 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -// syntax = "proto3"; @@ -43,3 +42,19 @@ message ArrowRecordBatch { // The count of rows in `serialized_record_batch`. int64 row_count = 2; } + +// Contains options specific to Arrow Serialization. +message ArrowSerializationOptions { + // Compression codec's supported by Arrow. + enum CompressionCodec { + // If unspecified no compression will be used. + COMPRESSION_UNSPECIFIED = 0; + + // LZ4 Frame (https://github.com/lz4/lz4/blob/dev/doc/lz4_Frame_format.md) + LZ4_FRAME = 1; + } + + // The compression codec to use for Arrow buffers in serialized record + // batches. + CompressionCodec buffer_compression = 2; +} diff --git a/google/cloud/bigquery_storage_v1/proto/avro.proto b/google/cloud/bigquery_storage_v1/proto/avro.proto index 9a064447..dee4a6ed 100644 --- a/google/cloud/bigquery_storage_v1/proto/avro.proto +++ b/google/cloud/bigquery_storage_v1/proto/avro.proto @@ -1,4 +1,4 @@ -// Copyright 2019 Google LLC. +// Copyright 2021 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -11,7 +11,6 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -// syntax = "proto3"; diff --git a/google/cloud/bigquery_storage_v1/proto/storage.proto b/google/cloud/bigquery_storage_v1/proto/storage.proto index 26fcd6ac..a5fa2b9e 100644 --- a/google/cloud/bigquery_storage_v1/proto/storage.proto +++ b/google/cloud/bigquery_storage_v1/proto/storage.proto @@ -1,4 +1,4 @@ -// Copyright 2019 Google LLC. +// Copyright 2021 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -11,7 +11,6 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -// syntax = "proto3"; @@ -70,7 +69,8 @@ service BigQueryRead { post: "/v1/{read_session.table=projects/*/datasets/*/tables/*}" body: "*" }; - option (google.api.method_signature) = "parent,read_session,max_stream_count"; + option (google.api.method_signature) = + "parent,read_session,max_stream_count"; } // Reads rows from the stream in the format prescribed by the ReadSession. @@ -99,7 +99,8 @@ service BigQueryRead { // original, primary, and residual, that original[0-j] = primary[0-j] and // original[j-n] = residual[0-m] once the streams have been read to // completion. - rpc SplitReadStream(SplitReadStreamRequest) returns (SplitReadStreamResponse) { + rpc SplitReadStream(SplitReadStreamRequest) + returns (SplitReadStreamResponse) { option (google.api.http) = { get: "/v1/{name=projects/*/locations/*/sessions/*/streams/*}" }; @@ -201,6 +202,19 @@ message ReadRowsResponse { // Throttling state. If unset, the latest response still describes // the current throttling status. ThrottleState throttle_state = 5; + + // The schema for the read. If read_options.selected_fields is set, the + // schema may be different from the table schema as it will only contain + // the selected fields. This schema is equivelant to the one returned by + // CreateSession. This field is only populated in the first ReadRowsResponse + // RPC. + oneof schema { + // Output only. Avro schema. + AvroSchema avro_schema = 7 [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. Arrow schema. + ArrowSchema arrow_schema = 8 [(google.api.field_behavior) = OUTPUT_ONLY]; + } } // Request message for `SplitReadStream`. diff --git a/google/cloud/bigquery_storage_v1/proto/stream.proto b/google/cloud/bigquery_storage_v1/proto/stream.proto index febad036..28b2ac1b 100644 --- a/google/cloud/bigquery_storage_v1/proto/stream.proto +++ b/google/cloud/bigquery_storage_v1/proto/stream.proto @@ -1,4 +1,4 @@ -// Copyright 2019 Google LLC. +// Copyright 2021 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -11,7 +11,6 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -// syntax = "proto3"; @@ -72,17 +71,27 @@ message ReadSession { // "nullable_field is not NULL" // "st_equals(geo_field, st_geofromtext("POINT(2, 2)"))" // "numeric_field BETWEEN 1.0 AND 5.0" + // + // Restricted to a maximum length for 1 MB. string row_restriction = 2; + + // Optional. Options specific to the Apache Arrow output format. + oneof output_format_serialization_options { + ArrowSerializationOptions arrow_serialization_options = 3 + [(google.api.field_behavior) = OPTIONAL]; + } } // Output only. Unique identifier for the session, in the form // `projects/{project_id}/locations/{location}/sessions/{session_id}`. string name = 1 [(google.api.field_behavior) = OUTPUT_ONLY]; - // Output only. Time at which the session becomes invalid. After this time, subsequent - // requests to read this Session will return errors. The expire_time is - // automatically assigned and currently cannot be specified or updated. - google.protobuf.Timestamp expire_time = 2 [(google.api.field_behavior) = OUTPUT_ONLY]; + // Output only. Time at which the session becomes invalid. After this time, + // subsequent requests to read this Session will return errors. The + // expire_time is automatically assigned and currently cannot be specified or + // updated. + google.protobuf.Timestamp expire_time = 2 + [(google.api.field_behavior) = OUTPUT_ONLY]; // Immutable. Data format of the output data. DataFormat data_format = 3 [(google.api.field_behavior) = IMMUTABLE]; @@ -102,12 +111,11 @@ message ReadSession { // `projects/{project_id}/datasets/{dataset_id}/tables/{table_id}` string table = 6 [ (google.api.field_behavior) = IMMUTABLE, - (google.api.resource_reference) = { - type: "bigquery.googleapis.com/Table" - } + (google.api.resource_reference) = { type: "bigquery.googleapis.com/Table" } ]; - // Optional. Any modifiers which are applied when reading from the specified table. + // Optional. Any modifiers which are applied when reading from the specified + // table. TableModifiers table_modifiers = 7 [(google.api.field_behavior) = OPTIONAL]; // Optional. Read options for this session (e.g. column selection, filters). diff --git a/google/cloud/bigquery_storage_v1/types/__init__.py b/google/cloud/bigquery_storage_v1/types/__init__.py index ee89843a..1b0763f0 100644 --- a/google/cloud/bigquery_storage_v1/types/__init__.py +++ b/google/cloud/bigquery_storage_v1/types/__init__.py @@ -18,6 +18,7 @@ from .arrow import ( ArrowRecordBatch, ArrowSchema, + ArrowSerializationOptions, ) from .avro import ( AvroRows, @@ -42,6 +43,7 @@ __all__ = ( "ArrowRecordBatch", "ArrowSchema", + "ArrowSerializationOptions", "AvroRows", "AvroSchema", "CreateReadSessionRequest", diff --git a/google/cloud/bigquery_storage_v1/types/arrow.py b/google/cloud/bigquery_storage_v1/types/arrow.py index e77b4576..1fe24e45 100644 --- a/google/cloud/bigquery_storage_v1/types/arrow.py +++ b/google/cloud/bigquery_storage_v1/types/arrow.py @@ -20,7 +20,7 @@ __protobuf__ = proto.module( package="google.cloud.bigquery.storage.v1", - manifest={"ArrowSchema", "ArrowRecordBatch",}, + manifest={"ArrowSchema", "ArrowRecordBatch", "ArrowSerializationOptions",}, ) @@ -55,4 +55,21 @@ class ArrowRecordBatch(proto.Message): row_count = proto.Field(proto.INT64, number=2) +class ArrowSerializationOptions(proto.Message): + r"""Contains options specific to Arrow Serialization. + + Attributes: + buffer_compression (google.cloud.bigquery_storage_v1.types.ArrowSerializationOptions.CompressionCodec): + The compression codec to use for Arrow + buffers in serialized record batches. + """ + + class CompressionCodec(proto.Enum): + r"""Compression codec's supported by Arrow.""" + COMPRESSION_UNSPECIFIED = 0 + LZ4_FRAME = 1 + + buffer_compression = proto.Field(proto.ENUM, number=2, enum=CompressionCodec,) + + __all__ = tuple(sorted(__protobuf__.manifest)) diff --git a/google/cloud/bigquery_storage_v1/types/storage.py b/google/cloud/bigquery_storage_v1/types/storage.py index 367d3bb3..7cd7766e 100644 --- a/google/cloud/bigquery_storage_v1/types/storage.py +++ b/google/cloud/bigquery_storage_v1/types/storage.py @@ -154,6 +154,10 @@ class ReadRowsResponse(proto.Message): Throttling state. If unset, the latest response still describes the current throttling status. + avro_schema (google.cloud.bigquery_storage_v1.types.AvroSchema): + Output only. Avro schema. + arrow_schema (google.cloud.bigquery_storage_v1.types.ArrowSchema): + Output only. Arrow schema. """ avro_rows = proto.Field( @@ -170,6 +174,14 @@ class ReadRowsResponse(proto.Message): throttle_state = proto.Field(proto.MESSAGE, number=5, message="ThrottleState",) + avro_schema = proto.Field( + proto.MESSAGE, number=7, oneof="schema", message=avro.AvroSchema, + ) + + arrow_schema = proto.Field( + proto.MESSAGE, number=8, oneof="schema", message=arrow.ArrowSchema, + ) + class SplitReadStreamRequest(proto.Message): r"""Request message for ``SplitReadStream``. diff --git a/google/cloud/bigquery_storage_v1/types/stream.py b/google/cloud/bigquery_storage_v1/types/stream.py index 34b865eb..4558bedc 100644 --- a/google/cloud/bigquery_storage_v1/types/stream.py +++ b/google/cloud/bigquery_storage_v1/types/stream.py @@ -104,13 +104,24 @@ class TableReadOptions(proto.Message): Examples: "int_field > 5" "date_field = CAST('2014-9-27' as DATE)" "nullable_field is not NULL" "st_equals(geo_field, st_geofromtext("POINT(2, 2)"))" "numeric_field BETWEEN 1.0 - AND 5.0". + AND 5.0" + + Restricted to a maximum length for 1 MB. + arrow_serialization_options (google.cloud.bigquery_storage_v1.types.ArrowSerializationOptions): + """ selected_fields = proto.RepeatedField(proto.STRING, number=1) row_restriction = proto.Field(proto.STRING, number=2) + arrow_serialization_options = proto.Field( + proto.MESSAGE, + number=3, + oneof="output_format_serialization_options", + message=arrow.ArrowSerializationOptions, + ) + name = proto.Field(proto.STRING, number=1) expire_time = proto.Field(proto.MESSAGE, number=2, message=timestamp.Timestamp,) diff --git a/synth.metadata b/synth.metadata index a2bc0bfc..cf7512cb 100644 --- a/synth.metadata +++ b/synth.metadata @@ -11,8 +11,8 @@ "git": { "name": "googleapis", "remote": "https://github.com/googleapis/googleapis.git", - "sha": "149a3a84c29c9b8189576c7442ccb6dcf6a8f95b", - "internalRef": "364411656" + "sha": "c539b9b08b3366ee00c0ec1950f4df711552a269", + "internalRef": "365759522" } }, { From 4a714f77542e8f6b7118fdd1d032ebee45c4f911 Mon Sep 17 00:00:00 2001 From: Yoshi Automation Bot Date: Wed, 31 Mar 2021 07:14:39 -0700 Subject: [PATCH 20/24] build(python): update docfx job to use new plugin (#171) Source-Author: Dan Lee <71398022+dandhlee@users.noreply.github.com> Source-Date: Tue Mar 30 19:36:37 2021 -0400 Source-Repo: googleapis/synthtool Source-Sha: 4501974ad08b5d693311457e2ea4ce845676e329 Source-Link: https://github.com/googleapis/synthtool/commit/4501974ad08b5d693311457e2ea4ce845676e329 --- noxfile.py | 6 +++--- synth.metadata | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/noxfile.py b/noxfile.py index 73ed1ac2..d775cb8a 100644 --- a/noxfile.py +++ b/noxfile.py @@ -218,9 +218,9 @@ def docfx(session): """Build the docfx yaml files for this library.""" session.install("-e", ".") - # sphinx-docfx-yaml supports up to sphinx version 1.5.5. - # https://github.com/docascode/sphinx-docfx-yaml/issues/97 - session.install("sphinx==1.5.5", "alabaster", "recommonmark", "sphinx-docfx-yaml") + session.install( + "sphinx<3.0.0", "alabaster", "recommonmark", "gcp-sphinx-docfx-yaml" + ) shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True) session.run( diff --git a/synth.metadata b/synth.metadata index cf7512cb..085fec9f 100644 --- a/synth.metadata +++ b/synth.metadata @@ -4,7 +4,7 @@ "git": { "name": ".", "remote": "https://github.com/googleapis/python-bigquery-storage.git", - "sha": "8c1071337ea6c76a61f601b46254e6f07d8f5add" + "sha": "1c91a276289a0e319f93b136836f81ee943f661c" } }, { @@ -19,14 +19,14 @@ "git": { "name": "synthtool", "remote": "https://github.com/googleapis/synthtool.git", - "sha": "86ed43d4f56e6404d068e62e497029018879c771" + "sha": "4501974ad08b5d693311457e2ea4ce845676e329" } }, { "git": { "name": "synthtool", "remote": "https://github.com/googleapis/synthtool.git", - "sha": "86ed43d4f56e6404d068e62e497029018879c771" + "sha": "4501974ad08b5d693311457e2ea4ce845676e329" } } ], From 3b6d86679a87aa6075d2578ba033eb67aa7b54b2 Mon Sep 17 00:00:00 2001 From: Yoshi Automation Bot Date: Wed, 7 Apr 2021 08:33:16 -0700 Subject: [PATCH 21/24] chore: Add license headers for python config files (#173) Source-Author: Anthonios Partheniou Source-Date: Tue Apr 6 11:32:03 2021 -0400 Source-Repo: googleapis/synthtool Source-Sha: 5b5bf6d519b2d658d9f2e483d9f6f3d0ba8ee6bc Source-Link: https://github.com/googleapis/synthtool/commit/5b5bf6d519b2d658d9f2e483d9f6f3d0ba8ee6bc --- .pre-commit-config.yaml | 14 ++++++++++++++ docs/conf.py | 13 +++++++++++++ synth.metadata | 6 +++--- 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 32302e48..8912e9b5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# # See https://pre-commit.com for more information # See https://pre-commit.com/hooks.html for more hooks repos: diff --git a/docs/conf.py b/docs/conf.py index 96d96148..6119caee 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,4 +1,17 @@ # -*- coding: utf-8 -*- +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # # google-cloud-bigquery-storage documentation build configuration file # diff --git a/synth.metadata b/synth.metadata index 085fec9f..0574a81a 100644 --- a/synth.metadata +++ b/synth.metadata @@ -4,7 +4,7 @@ "git": { "name": ".", "remote": "https://github.com/googleapis/python-bigquery-storage.git", - "sha": "1c91a276289a0e319f93b136836f81ee943f661c" + "sha": "4a714f77542e8f6b7118fdd1d032ebee45c4f911" } }, { @@ -19,14 +19,14 @@ "git": { "name": "synthtool", "remote": "https://github.com/googleapis/synthtool.git", - "sha": "4501974ad08b5d693311457e2ea4ce845676e329" + "sha": "5b5bf6d519b2d658d9f2e483d9f6f3d0ba8ee6bc" } }, { "git": { "name": "synthtool", "remote": "https://github.com/googleapis/synthtool.git", - "sha": "4501974ad08b5d693311457e2ea4ce845676e329" + "sha": "5b5bf6d519b2d658d9f2e483d9f6f3d0ba8ee6bc" } } ], From 97998be32a963d98cc0e6a3cb7609ae07f1eb4c2 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Wed, 7 Apr 2021 17:34:43 +0200 Subject: [PATCH 22/24] chore(deps): update dependency google-cloud-bigquery to v2.13.1 (#163) --- samples/to_dataframe/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/to_dataframe/requirements.txt b/samples/to_dataframe/requirements.txt index 29a45256..c5bae2eb 100644 --- a/samples/to_dataframe/requirements.txt +++ b/samples/to_dataframe/requirements.txt @@ -1,6 +1,6 @@ google-auth==1.28.0 google-cloud-bigquery-storage==2.3.0 -google-cloud-bigquery==2.12.0 +google-cloud-bigquery==2.13.1 pyarrow==3.0.0 ipython==7.10.2; python_version > '3.0' ipython==5.9.0; python_version < '3.0' From bef63fbb3b7e41e1c0d73f91a2c86d4d24e42151 Mon Sep 17 00:00:00 2001 From: Yoshi Automation Bot Date: Wed, 7 Apr 2021 08:59:13 -0700 Subject: [PATCH 23/24] feat: updates for v1beta2 storage API - Updated comments on BatchCommitWriteStreams - Added new support Bigquery types BIGNUMERIC and INTERVAL to TableSchema - Added read rows schema in ReadRowsResponse - Misc comment updates (#172) Committer: @yirutang PiperOrigin-RevId: 366811078 Source-Author: Google APIs Source-Date: Mon Apr 5 09:19:17 2021 -0700 Source-Repo: googleapis/googleapis Source-Sha: b1614aa0668564ec41d78b72cf776e0292ffc98c Source-Link: https://github.com/googleapis/googleapis/commit/b1614aa0668564ec41d78b72cf776e0292ffc98c --- .../proto/arrow.proto | 2 +- .../bigquery_storage_v1beta2/proto/avro.proto | 2 +- .../proto/protobuf.proto | 5 +- .../proto/storage.proto | 100 +++++++++++------- .../proto/stream.proto | 4 +- .../proto/table.proto | 8 +- .../types/protobuf.py | 6 +- .../bigquery_storage_v1beta2/types/storage.py | 73 +++++++++---- .../bigquery_storage_v1beta2/types/stream.py | 4 +- .../bigquery_storage_v1beta2/types/table.py | 2 + synth.metadata | 4 +- 11 files changed, 139 insertions(+), 71 deletions(-) diff --git a/google/cloud/bigquery_storage_v1beta2/proto/arrow.proto b/google/cloud/bigquery_storage_v1beta2/proto/arrow.proto index bc2e4eb1..74fe927b 100644 --- a/google/cloud/bigquery_storage_v1beta2/proto/arrow.proto +++ b/google/cloud/bigquery_storage_v1beta2/proto/arrow.proto @@ -1,4 +1,4 @@ -// Copyright 2020 Google LLC +// Copyright 2021 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/google/cloud/bigquery_storage_v1beta2/proto/avro.proto b/google/cloud/bigquery_storage_v1beta2/proto/avro.proto index 109ec86a..495132ec 100644 --- a/google/cloud/bigquery_storage_v1beta2/proto/avro.proto +++ b/google/cloud/bigquery_storage_v1beta2/proto/avro.proto @@ -1,4 +1,4 @@ -// Copyright 2020 Google LLC +// Copyright 2021 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/google/cloud/bigquery_storage_v1beta2/proto/protobuf.proto b/google/cloud/bigquery_storage_v1beta2/proto/protobuf.proto index 741e7d11..11e851be 100644 --- a/google/cloud/bigquery_storage_v1beta2/proto/protobuf.proto +++ b/google/cloud/bigquery_storage_v1beta2/proto/protobuf.proto @@ -1,4 +1,4 @@ -// Copyright 2020 Google LLC +// Copyright 2021 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -23,7 +23,7 @@ option java_multiple_files = true; option java_outer_classname = "ProtoBufProto"; option java_package = "com.google.cloud.bigquery.storage.v1beta2"; -// Protobuf schema is an API presentation the proto buffer schema. +// ProtoSchema describes the schema of the serialized protocol buffer data rows. message ProtoSchema { // Descriptor for input message. The descriptor has to be self contained, // including all the nested types, excepted for proto buffer well known types @@ -31,7 +31,6 @@ message ProtoSchema { google.protobuf.DescriptorProto proto_descriptor = 1; } -// Protobuf rows. message ProtoRows { // A sequence of rows serialized as a Protocol Buffer. // diff --git a/google/cloud/bigquery_storage_v1beta2/proto/storage.proto b/google/cloud/bigquery_storage_v1beta2/proto/storage.proto index 5538e29f..8c25b846 100644 --- a/google/cloud/bigquery_storage_v1beta2/proto/storage.proto +++ b/google/cloud/bigquery_storage_v1beta2/proto/storage.proto @@ -1,4 +1,4 @@ -// Copyright 2020 Google LLC +// Copyright 2021 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -71,8 +71,7 @@ service BigQueryRead { post: "/v1beta2/{read_session.table=projects/*/datasets/*/tables/*}" body: "*" }; - option (google.api.method_signature) = - "parent,read_session,max_stream_count"; + option (google.api.method_signature) = "parent,read_session,max_stream_count"; } // Reads rows from the stream in the format prescribed by the ReadSession. @@ -101,8 +100,7 @@ service BigQueryRead { // original, primary, and residual, that original[0-j] = primary[0-j] and // original[j-n] = residual[0-m] once the streams have been read to // completion. - rpc SplitReadStream(SplitReadStreamRequest) - returns (SplitReadStreamResponse) { + rpc SplitReadStream(SplitReadStreamRequest) returns (SplitReadStreamResponse) { option (google.api.http) = { get: "/v1beta2/{name=projects/*/locations/*/sessions/*/streams/*}" }; @@ -171,8 +169,7 @@ service BigQueryWrite { // Finalize a write stream so that no new data can be appended to the // stream. Finalize is not supported on the '_default' stream. - rpc FinalizeWriteStream(FinalizeWriteStreamRequest) - returns (FinalizeWriteStreamResponse) { + rpc FinalizeWriteStream(FinalizeWriteStreamRequest) returns (FinalizeWriteStreamResponse) { option (google.api.http) = { post: "/v1beta2/{name=projects/*/datasets/*/tables/*/streams/*}" body: "*" @@ -185,8 +182,7 @@ service BigQueryWrite { // Streams must be finalized before commit and cannot be committed multiple // times. Once a stream is committed, data in the stream becomes available // for read operations. - rpc BatchCommitWriteStreams(BatchCommitWriteStreamsRequest) - returns (BatchCommitWriteStreamsResponse) { + rpc BatchCommitWriteStreams(BatchCommitWriteStreamsRequest) returns (BatchCommitWriteStreamsResponse) { option (google.api.http) = { get: "/v1beta2/{parent=projects/*/datasets/*/tables/*}" }; @@ -303,6 +299,19 @@ message ReadRowsResponse { // Throttling state. If unset, the latest response still describes // the current throttling status. ThrottleState throttle_state = 5; + + // The schema for the read. If read_options.selected_fields is set, the + // schema may be different from the table schema as it will only contain + // the selected fields. This schema is equivelant to the one returned by + // CreateSession. This field is only populated in the first ReadRowsResponse + // RPC. + oneof schema { + // Output only. Avro schema. + AvroSchema avro_schema = 7 [(google.api.field_behavior) = OUTPUT_ONLY]; + + // Output only. Arrow schema. + ArrowSchema arrow_schema = 8 [(google.api.field_behavior) = OUTPUT_ONLY]; + } } // Request message for `SplitReadStream`. @@ -342,7 +351,9 @@ message CreateWriteStreamRequest { // of `projects/{project}/datasets/{dataset}/tables/{table}`. string parent = 1 [ (google.api.field_behavior) = REQUIRED, - (google.api.resource_reference) = { type: "bigquery.googleapis.com/Table" } + (google.api.resource_reference) = { + type: "bigquery.googleapis.com/Table" + } ]; // Required. Stream to be created. @@ -360,9 +371,9 @@ message AppendRowsRequest { ProtoRows rows = 2; } - // Required. The stream that is the target of the append operation. This value - // must be specified for the initial request. If subsequent requests specify - // the stream name, it must equal to the value provided in the first request. + // Required. The stream that is the target of the append operation. This value must be + // specified for the initial request. If subsequent requests specify the + // stream name, it must equal to the value provided in the first request. // To write to the _default stream, populate this field with a string in the // format `projects/{project}/datasets/{dataset}/tables/{table}/_default`. string write_stream = 1 [ @@ -394,7 +405,7 @@ message AppendRowsRequest { // Response message for `AppendRows`. message AppendRowsResponse { - // A success append result. + // AppendResult is returned for successful append requests. message AppendResult { // The row offset at which the last append occurred. The offset will not be // set if appending using default streams. @@ -405,25 +416,32 @@ message AppendRowsResponse { // Result if the append is successful. AppendResult append_result = 1; - // Error in case of request failed. If set, it means rows are not accepted - // into the system. Users can retry or continue with other requests within - // the same connection. - // ALREADY_EXISTS: happens when offset is specified, it means the entire - // request is already appended, it is safe to ignore this error. - // OUT_OF_RANGE: happens when offset is specified, it means the specified - // offset is beyond the end of the stream. - // INVALID_ARGUMENT: error caused by malformed request or data. - // RESOURCE_EXHAUSTED: request rejected due to throttling. Only happens when - // append without offset. - // ABORTED: request processing is aborted because of prior failures, request - // can be retried if previous failure is fixed. - // INTERNAL: server side errors that can be retried. + // Error returned when problems were encountered. If present, + // it indicates rows were not accepted into the system. + // Users can retry or continue with other append requests within the + // same connection. + // + // Additional information about error signalling: + // + // ALREADY_EXISTS: Happens when an append specified an offset, and the + // backend already has received data at this offset. Typically encountered + // in retry scenarios, and can be ignored. + // + // OUT_OF_RANGE: Returned when the specified offset in the stream is beyond + // the current end of the stream. + // + // INVALID_ARGUMENT: Indicates a malformed request or data. + // + // ABORTED: Request processing is aborted because of prior failures. The + // request can be retried if previous failure is addressed. + // + // INTERNAL: Indicates server side error(s) that can be retried. google.rpc.Status error = 2; } // If backend detects a schema update, pass it to user so that user can - // use it to input new type of message. It will be empty when there is no - // schema updates. + // use it to input new type of message. It will be empty when no schema + // updates have occurred. TableSchema updated_schema = 3; } @@ -441,9 +459,11 @@ message GetWriteStreamRequest { // Request message for `BatchCommitWriteStreams`. message BatchCommitWriteStreamsRequest { - // Required. Parent table that all the streams should belong to, in the form - // of `projects/{project}/datasets/{dataset}/tables/{table}`. - string parent = 1 [(google.api.field_behavior) = REQUIRED]; + // Required. Parent table that all the streams should belong to, in the form of + // `projects/{project}/datasets/{dataset}/tables/{table}`. + string parent = 1 [ + (google.api.field_behavior) = REQUIRED + ]; // Required. The group of streams that will be committed atomically. repeated string write_streams = 2 [(google.api.field_behavior) = REQUIRED]; @@ -452,11 +472,15 @@ message BatchCommitWriteStreamsRequest { // Response message for `BatchCommitWriteStreams`. message BatchCommitWriteStreamsResponse { // The time at which streams were committed in microseconds granularity. - // This field will only exist when there is no stream errors. + // This field will only exist when there are no stream errors. + // **Note** if this field is not set, it means the commit was not successful. google.protobuf.Timestamp commit_time = 1; // Stream level error if commit failed. Only streams with error will be in // the list. + // If empty, there is no error and all streams are committed successfully. + // If non empty, certain streams have errors and ZERO stream is committed due + // to atomicity guarantee. repeated StorageError stream_errors = 2; } @@ -500,8 +524,9 @@ message FlushRowsResponse { } // Structured custom BigQuery Storage error message. The error can be attached -// as error details in the returned rpc Status. User can use the info to process -// errors in a structural way, rather than having to parse error messages. +// as error details in the returned rpc Status. In particular, the use of error +// codes allows more structured error handling, and reduces the need to evaluate +// unstructured error text strings. message StorageError { // Error code for `StorageError`. enum StorageErrorCode { @@ -522,9 +547,12 @@ message StorageError { INVALID_STREAM_TYPE = 4; // Invalid Stream state. - // For example, you try to commit a stream that is not fianlized or is + // For example, you try to commit a stream that is not finalized or is // garbaged. INVALID_STREAM_STATE = 5; + + // Stream is finalized. + STREAM_FINALIZED = 6; } // BigQuery Storage specific error code. diff --git a/google/cloud/bigquery_storage_v1beta2/proto/stream.proto b/google/cloud/bigquery_storage_v1beta2/proto/stream.proto index 2b0a58c9..d166e987 100644 --- a/google/cloud/bigquery_storage_v1beta2/proto/stream.proto +++ b/google/cloud/bigquery_storage_v1beta2/proto/stream.proto @@ -1,4 +1,4 @@ -// Copyright 2020 Google LLC +// Copyright 2021 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -74,6 +74,8 @@ message ReadSession { // "nullable_field is not NULL" // "st_equals(geo_field, st_geofromtext("POINT(2, 2)"))" // "numeric_field BETWEEN 1.0 AND 5.0" + // + // Restricted to a maximum length for 1 MB. string row_restriction = 2; // Optional. Options specific to the Apache Arrow output format. diff --git a/google/cloud/bigquery_storage_v1beta2/proto/table.proto b/google/cloud/bigquery_storage_v1beta2/proto/table.proto index fd8a0a75..670a4a64 100644 --- a/google/cloud/bigquery_storage_v1beta2/proto/table.proto +++ b/google/cloud/bigquery_storage_v1beta2/proto/table.proto @@ -1,4 +1,4 @@ -// Copyright 2020 Google LLC +// Copyright 2021 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -70,6 +70,12 @@ message TableFieldSchema { // Numeric value NUMERIC = 12; + + // BigNumeric value + BIGNUMERIC = 13; + + // Interval + INTERVAL = 14; } enum Mode { diff --git a/google/cloud/bigquery_storage_v1beta2/types/protobuf.py b/google/cloud/bigquery_storage_v1beta2/types/protobuf.py index 99c0543c..9b88202f 100644 --- a/google/cloud/bigquery_storage_v1beta2/types/protobuf.py +++ b/google/cloud/bigquery_storage_v1beta2/types/protobuf.py @@ -28,8 +28,8 @@ class ProtoSchema(proto.Message): - r"""Protobuf schema is an API presentation the proto buffer - schema. + r"""ProtoSchema describes the schema of the serialized protocol + buffer data rows. Attributes: proto_descriptor (google.protobuf.descriptor_pb2.DescriptorProto): @@ -47,7 +47,7 @@ class ProtoSchema(proto.Message): class ProtoRows(proto.Message): - r"""Protobuf rows. + r""" Attributes: serialized_rows (Sequence[bytes]): diff --git a/google/cloud/bigquery_storage_v1beta2/types/storage.py b/google/cloud/bigquery_storage_v1beta2/types/storage.py index 85369388..ee6d5210 100644 --- a/google/cloud/bigquery_storage_v1beta2/types/storage.py +++ b/google/cloud/bigquery_storage_v1beta2/types/storage.py @@ -170,6 +170,10 @@ class ReadRowsResponse(proto.Message): Throttling state. If unset, the latest response still describes the current throttling status. + avro_schema (google.cloud.bigquery_storage_v1beta2.types.AvroSchema): + Output only. Avro schema. + arrow_schema (google.cloud.bigquery_storage_v1beta2.types.ArrowSchema): + Output only. Arrow schema. """ avro_rows = proto.Field( @@ -186,6 +190,14 @@ class ReadRowsResponse(proto.Message): throttle_state = proto.Field(proto.MESSAGE, number=5, message="ThrottleState",) + avro_schema = proto.Field( + proto.MESSAGE, number=7, oneof="schema", message=avro.AvroSchema, + ) + + arrow_schema = proto.Field( + proto.MESSAGE, number=8, oneof="schema", message=arrow.ArrowSchema, + ) + class SplitReadStreamRequest(proto.Message): r"""Request message for ``SplitReadStream``. @@ -307,28 +319,38 @@ class AppendRowsResponse(proto.Message): append_result (google.cloud.bigquery_storage_v1beta2.types.AppendRowsResponse.AppendResult): Result if the append is successful. error (google.rpc.status_pb2.Status): - Error in case of request failed. If set, it means rows are - not accepted into the system. Users can retry or continue - with other requests within the same connection. - ALREADY_EXISTS: happens when offset is specified, it means - the entire request is already appended, it is safe to ignore - this error. OUT_OF_RANGE: happens when offset is specified, - it means the specified offset is beyond the end of the - stream. INVALID_ARGUMENT: error caused by malformed request - or data. RESOURCE_EXHAUSTED: request rejected due to - throttling. Only happens when append without offset. - ABORTED: request processing is aborted because of prior - failures, request can be retried if previous failure is - fixed. INTERNAL: server side errors that can be retried. + Error returned when problems were encountered. If present, + it indicates rows were not accepted into the system. Users + can retry or continue with other append requests within the + same connection. + + Additional information about error signalling: + + ALREADY_EXISTS: Happens when an append specified an offset, + and the backend already has received data at this offset. + Typically encountered in retry scenarios, and can be + ignored. + + OUT_OF_RANGE: Returned when the specified offset in the + stream is beyond the current end of the stream. + + INVALID_ARGUMENT: Indicates a malformed request or data. + + ABORTED: Request processing is aborted because of prior + failures. The request can be retried if previous failure is + addressed. + + INTERNAL: Indicates server side error(s) that can be + retried. updated_schema (google.cloud.bigquery_storage_v1beta2.types.TableSchema): If backend detects a schema update, pass it to user so that user can use it to input new - type of message. It will be empty when there is - no schema updates. + type of message. It will be empty when no schema + updates have occurred. """ class AppendResult(proto.Message): - r"""A success append result. + r"""AppendResult is returned for successful append requests. Attributes: offset (google.protobuf.wrappers_pb2.Int64Value): @@ -385,12 +407,17 @@ class BatchCommitWriteStreamsResponse(proto.Message): Attributes: commit_time (google.protobuf.timestamp_pb2.Timestamp): - The time at which streams were committed in - microseconds granularity. This field will only - exist when there is no stream errors. + The time at which streams were committed in microseconds + granularity. This field will only exist when there are no + stream errors. **Note** if this field is not set, it means + the commit was not successful. stream_errors (Sequence[google.cloud.bigquery_storage_v1beta2.types.StorageError]): Stream level error if commit failed. Only streams with error will be in the list. + If empty, there is no error and all streams are + committed successfully. If non empty, certain + streams have errors and ZERO stream is committed + due to atomicity guarantee. """ commit_time = proto.Field(proto.MESSAGE, number=1, message=timestamp.Timestamp,) @@ -455,9 +482,10 @@ class FlushRowsResponse(proto.Message): class StorageError(proto.Message): r"""Structured custom BigQuery Storage error message. The error - can be attached as error details in the returned rpc Status. - User can use the info to process errors in a structural way, - rather than having to parse error messages. + can be attached as error details in the returned rpc Status. In + particular, the use of error codes allows more structured error + handling, and reduces the need to evaluate unstructured error + text strings. Attributes: code (google.cloud.bigquery_storage_v1beta2.types.StorageError.StorageErrorCode): @@ -476,6 +504,7 @@ class StorageErrorCode(proto.Enum): STREAM_NOT_FOUND = 3 INVALID_STREAM_TYPE = 4 INVALID_STREAM_STATE = 5 + STREAM_FINALIZED = 6 code = proto.Field(proto.ENUM, number=1, enum=StorageErrorCode,) diff --git a/google/cloud/bigquery_storage_v1beta2/types/stream.py b/google/cloud/bigquery_storage_v1beta2/types/stream.py index 77fb444c..e635e5a6 100644 --- a/google/cloud/bigquery_storage_v1beta2/types/stream.py +++ b/google/cloud/bigquery_storage_v1beta2/types/stream.py @@ -105,7 +105,9 @@ class TableReadOptions(proto.Message): Examples: "int_field > 5" "date_field = CAST('2014-9-27' as DATE)" "nullable_field is not NULL" "st_equals(geo_field, st_geofromtext("POINT(2, 2)"))" "numeric_field BETWEEN 1.0 - AND 5.0". + AND 5.0" + + Restricted to a maximum length for 1 MB. arrow_serialization_options (google.cloud.bigquery_storage_v1beta2.types.ArrowSerializationOptions): Optional. Options specific to the Apache Arrow output format. diff --git a/google/cloud/bigquery_storage_v1beta2/types/table.py b/google/cloud/bigquery_storage_v1beta2/types/table.py index f1a209e0..f91b07b4 100644 --- a/google/cloud/bigquery_storage_v1beta2/types/table.py +++ b/google/cloud/bigquery_storage_v1beta2/types/table.py @@ -72,6 +72,8 @@ class Type(proto.Enum): DATETIME = 10 GEOGRAPHY = 11 NUMERIC = 12 + BIGNUMERIC = 13 + INTERVAL = 14 class Mode(proto.Enum): r"""""" diff --git a/synth.metadata b/synth.metadata index 0574a81a..1eec6cb9 100644 --- a/synth.metadata +++ b/synth.metadata @@ -11,8 +11,8 @@ "git": { "name": "googleapis", "remote": "https://github.com/googleapis/googleapis.git", - "sha": "c539b9b08b3366ee00c0ec1950f4df711552a269", - "internalRef": "365759522" + "sha": "b1614aa0668564ec41d78b72cf776e0292ffc98c", + "internalRef": "366811078" } }, { From d96d39a532c58c857aafe02ff54a0d3ce195adb4 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Wed, 7 Apr 2021 16:14:03 +0000 Subject: [PATCH 24/24] chore: release 2.4.0 (#170) :robot: I have created a release \*beep\* \*boop\* --- ## [2.4.0](https://www.github.com/googleapis/python-bigquery-storage/compare/v2.3.0...v2.4.0) (2021-04-07) ### Features * add a Arrow compression options (Only LZ4 for now) ([#166](https://www.github.com/googleapis/python-bigquery-storage/issues/166)) ([1c91a27](https://www.github.com/googleapis/python-bigquery-storage/commit/1c91a276289a0e319f93b136836f81ee943f661c)) * updates for v1beta2 storage API - Updated comments on BatchCommitWriteStreams - Added new support Bigquery types BIGNUMERIC and INTERVAL to TableSchema - Added read rows schema in ReadRowsResponse - Misc comment updates ([#172](https://www.github.com/googleapis/python-bigquery-storage/issues/172)) ([bef63fb](https://www.github.com/googleapis/python-bigquery-storage/commit/bef63fbb3b7e41e1c0d73f91a2c86d4d24e42151)) ### Dependencies * update minimum pandas to 0.21.1 ([#165](https://www.github.com/googleapis/python-bigquery-storage/issues/165)) ([8a97763](https://www.github.com/googleapis/python-bigquery-storage/commit/8a977633a81d080f03f6922752adbf4284199dd4)) --- This PR was generated with [Release Please](https://github.com/googleapis/release-please). See [documentation](https://github.com/googleapis/release-please#release-please). --- CHANGELOG.md | 13 +++++++++++++ setup.py | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 69fe6ffb..df7e6cce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,19 @@ [1]: https://pypi.org/project/google-cloud-bigquery-storage/#history +## [2.4.0](https://www.github.com/googleapis/python-bigquery-storage/compare/v2.3.0...v2.4.0) (2021-04-07) + + +### Features + +* add a Arrow compression options (Only LZ4 for now) ([#166](https://www.github.com/googleapis/python-bigquery-storage/issues/166)) ([1c91a27](https://www.github.com/googleapis/python-bigquery-storage/commit/1c91a276289a0e319f93b136836f81ee943f661c)) +* updates for v1beta2 storage API - Updated comments on BatchCommitWriteStreams - Added new support Bigquery types BIGNUMERIC and INTERVAL to TableSchema - Added read rows schema in ReadRowsResponse - Misc comment updates ([#172](https://www.github.com/googleapis/python-bigquery-storage/issues/172)) ([bef63fb](https://www.github.com/googleapis/python-bigquery-storage/commit/bef63fbb3b7e41e1c0d73f91a2c86d4d24e42151)) + + +### Dependencies + +* update minimum pandas to 0.21.1 ([#165](https://www.github.com/googleapis/python-bigquery-storage/issues/165)) ([8a97763](https://www.github.com/googleapis/python-bigquery-storage/commit/8a977633a81d080f03f6922752adbf4284199dd4)) + ## [2.3.0](https://www.github.com/googleapis/python-bigquery-storage/compare/v2.2.1...v2.3.0) (2021-02-18) diff --git a/setup.py b/setup.py index e1896bd1..8de1c6c2 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ name = "google-cloud-bigquery-storage" description = "BigQuery Storage API API client library" -version = "2.3.0" +version = "2.4.0" release_status = "Development Status :: 5 - Production/Stable" dependencies = [ "google-api-core[grpc] >= 1.22.2, < 2.0.0dev",