Skip to content

Commit 898bd61

Browse files
gcf-owl-bot[bot]parthea
authored andcommitted
docs: clarify size limitations for AppendRowsRequest (#474)
* docs: clarify size limitations for AppendRowsRequest chore: add preferred_min_stream_count to CreateReadSessionRequest chore: add write_stream to AppendRowsResponse PiperOrigin-RevId: 463602530 Source-Link: googleapis/googleapis@d33b3fa Source-Link: googleapis/googleapis-gen@90995f6 Copy-Tag: eyJwIjoiLmdpdGh1Yi8uT3dsQm90LnlhbWwiLCJoIjoiOTA5OTVmNjQzM2QwZWNkMjkwZjE4NjE2OGNlOTU3ZDZhMGRiOWM2OCJ9 * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>
1 parent d845afd commit 898bd61

4 files changed

Lines changed: 34 additions & 1 deletion

File tree

packages/google-cloud-bigquery-storage/google/cloud/bigquery_storage_v1/services/big_query_write/async_client.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -431,6 +431,9 @@ def request_generator():
431431
streaming RPC, certain parts of the AppendRowsRequest
432432
need only be specified for the first request sent each
433433
time the gRPC network connection is opened/reopened.
434+
The size of a single AppendRowsRequest must be less than
435+
10 MB in size. Requests larger than this return an
436+
error, typically `INVALID_ARGUMENT`.
434437
retry (google.api_core.retry.Retry): Designation of what errors, if any,
435438
should be retried.
436439
timeout (float): The timeout for this request.

packages/google-cloud-bigquery-storage/google/cloud/bigquery_storage_v1/services/big_query_write/client.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -664,6 +664,9 @@ def request_generator():
664664
streaming RPC, certain parts of the AppendRowsRequest
665665
need only be specified for the first request sent each
666666
time the gRPC network connection is opened/reopened.
667+
The size of a single AppendRowsRequest must be less than
668+
10 MB in size. Requests larger than this return an
669+
error, typically `INVALID_ARGUMENT`.
667670
retry (google.api_core.retry.Retry): Designation of what errors, if any,
668671
should be retried.
669672
timeout (float): The timeout for this request.

packages/google-cloud-bigquery-storage/google/cloud/bigquery_storage_v1/types/storage.py

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,18 @@ class CreateReadSessionRequest(proto.Message):
7373
leave this unset to let the system to determine an upper
7474
bound OR set this a size for the maximum "units of work" it
7575
can gracefully handle.
76+
preferred_min_stream_count (int):
77+
The minimum preferred stream count. This
78+
parameter can be used to inform the service that
79+
there is a desired lower bound on the number of
80+
streams. This is typically a target parallelism
81+
of the client (e.g. a Spark cluster with
82+
N-workers would set this to a low multiple of N
83+
to ensure good cluster utilization).
84+
85+
The system will make a best effort to provide at
86+
least this number of streams, but in some cases
87+
might provide less.
7688
"""
7789

7890
parent = proto.Field(
@@ -88,6 +100,10 @@ class CreateReadSessionRequest(proto.Message):
88100
proto.INT32,
89101
number=3,
90102
)
103+
preferred_min_stream_count = proto.Field(
104+
proto.INT32,
105+
number=4,
106+
)
91107

92108

93109
class ReadRowsRequest(proto.Message):
@@ -343,6 +359,10 @@ class AppendRowsRequest(proto.Message):
343359
the first request sent each time the gRPC network connection is
344360
opened/reopened.
345361
362+
The size of a single AppendRowsRequest must be less than 10 MB in
363+
size. Requests larger than this return an error, typically
364+
``INVALID_ARGUMENT``.
365+
346366
347367
.. _oneof: https://proto-plus-python.readthedocs.io/en/stable/fields.html#oneofs-mutually-exclusive-fields
348368
@@ -478,6 +498,9 @@ class AppendRowsResponse(proto.Message):
478498
rows in the batch will be appended. The API will
479499
return row level error info, so that the caller
480500
can remove the bad rows and retry the request.
501+
write_stream (str):
502+
The target of the append operation. Matches the write_stream
503+
in the corresponding request.
481504
"""
482505

483506
class AppendResult(proto.Message):
@@ -518,6 +541,10 @@ class AppendResult(proto.Message):
518541
number=4,
519542
message="RowError",
520543
)
544+
write_stream = proto.Field(
545+
proto.STRING,
546+
number=5,
547+
)
521548

522549

523550
class GetWriteStreamRequest(proto.Message):

packages/google-cloud-bigquery-storage/scripts/fixup_bigquery_storage_v1_keywords.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ class bigquery_storageCallTransformer(cst.CSTTransformer):
4141
METHOD_TO_PARAMS: Dict[str, Tuple[str]] = {
4242
'append_rows': ('write_stream', 'offset', 'proto_rows', 'trace_id', ),
4343
'batch_commit_write_streams': ('parent', 'write_streams', ),
44-
'create_read_session': ('parent', 'read_session', 'max_stream_count', ),
44+
'create_read_session': ('parent', 'read_session', 'max_stream_count', 'preferred_min_stream_count', ),
4545
'create_write_stream': ('parent', 'write_stream', ),
4646
'finalize_write_stream': ('name', ),
4747
'flush_rows': ('write_stream', 'offset', ),

0 commit comments

Comments
 (0)