Skip to content

Commit edfc3f6

Browse files
yoshi-automationtseaver
yoshi-automation
authored and
tseaver
committed
feat(dataproc): add V1 autoscaling policy support; annotate protos (via synth) (googleapis#9402)
1 parent 9ea608d commit edfc3f6

36 files changed

Lines changed: 5327 additions & 1689 deletions

‎dataproc/docs/conf.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -344,7 +344,7 @@
344344
"google-gax": ("https://gax-python.readthedocs.io/en/latest/", None),
345345
"google.api_core": ("https://googleapis.dev/python/google-api-core/latest", None),
346346
"grpc": ("https://grpc.io/grpc/python/", None),
347-
"requests": ("https://requests.kennethreitz.org/en/stable/", None),
347+
"requests": ("https://requests.kennethreitz.org/en/master/", None),
348348
"fastavro": ("https://fastavro.readthedocs.io/en/stable/", None),
349349
"pandas": ("https://pandas.pydata.org/pandas-docs/stable/", None),
350350
}

‎dataproc/google/cloud/dataproc_v1/gapic/cluster_controller_client.py‎

Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -203,7 +203,9 @@ def create_cluster(
203203
metadata=None,
204204
):
205205
"""
206-
Creates a cluster in a project.
206+
Creates a cluster in a project. The returned ``Operation.metadata`` will
207+
be
208+
`ClusterOperationMetadata <https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#clusteroperationmetadata>`__.
207209
208210
Example:
209211
>>> from google.cloud import dataproc_v1
@@ -306,7 +308,9 @@ def update_cluster(
306308
metadata=None,
307309
):
308310
"""
309-
Updates a cluster in a project.
311+
Updates a cluster in a project. The returned ``Operation.metadata`` will
312+
be
313+
`ClusterOperationMetadata <https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#clusteroperationmetadata>`__.
310314
311315
Example:
312316
>>> from google.cloud import dataproc_v1
@@ -401,6 +405,10 @@ def update_cluster(
401405
<td><strong><em>config.secondary_worker_config.num_instances</em></strong></td>
402406
<td>Resize secondary worker group</td>
403407
</tr>
408+
<tr>
409+
<td>config.autoscaling_config.policy_uri</td><td>Use, stop using, or
410+
change autoscaling policies</td>
411+
</tr>
404412
</tbody>
405413
</table>
406414
@@ -489,7 +497,9 @@ def delete_cluster(
489497
metadata=None,
490498
):
491499
"""
492-
Deletes a cluster in a project.
500+
Deletes a cluster in a project. The returned ``Operation.metadata`` will
501+
be
502+
`ClusterOperationMetadata <https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#clusteroperationmetadata>`__.
493503
494504
Example:
495505
>>> from google.cloud import dataproc_v1
@@ -779,8 +789,11 @@ def diagnose_cluster(
779789
metadata=None,
780790
):
781791
"""
782-
Gets cluster diagnostic information. After the operation completes, the
783-
Operation.response field contains ``DiagnoseClusterOutputLocation``.
792+
Gets cluster diagnostic information. The returned ``Operation.metadata``
793+
will be
794+
`ClusterOperationMetadata <https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#clusteroperationmetadata>`__.
795+
After the operation completes, ``Operation.response`` contains
796+
`DiagnoseClusterResults <https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#diagnoseclusterresults>`__.
784797
785798
Example:
786799
>>> from google.cloud import dataproc_v1

‎dataproc/google/cloud/dataproc_v1/gapic/transports/cluster_controller_grpc_transport.py‎

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -119,7 +119,9 @@ def channel(self):
119119
def create_cluster(self):
120120
"""Return the gRPC stub for :meth:`ClusterControllerClient.create_cluster`.
121121
122-
Creates a cluster in a project.
122+
Creates a cluster in a project. The returned ``Operation.metadata`` will
123+
be
124+
`ClusterOperationMetadata <https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#clusteroperationmetadata>`__.
123125
124126
Returns:
125127
Callable: A callable which accepts the appropriate
@@ -132,7 +134,9 @@ def create_cluster(self):
132134
def update_cluster(self):
133135
"""Return the gRPC stub for :meth:`ClusterControllerClient.update_cluster`.
134136
135-
Updates a cluster in a project.
137+
Updates a cluster in a project. The returned ``Operation.metadata`` will
138+
be
139+
`ClusterOperationMetadata <https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#clusteroperationmetadata>`__.
136140
137141
Returns:
138142
Callable: A callable which accepts the appropriate
@@ -145,7 +149,9 @@ def update_cluster(self):
145149
def delete_cluster(self):
146150
"""Return the gRPC stub for :meth:`ClusterControllerClient.delete_cluster`.
147151
148-
Deletes a cluster in a project.
152+
Deletes a cluster in a project. The returned ``Operation.metadata`` will
153+
be
154+
`ClusterOperationMetadata <https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#clusteroperationmetadata>`__.
149155
150156
Returns:
151157
Callable: A callable which accepts the appropriate
@@ -184,8 +190,11 @@ def list_clusters(self):
184190
def diagnose_cluster(self):
185191
"""Return the gRPC stub for :meth:`ClusterControllerClient.diagnose_cluster`.
186192
187-
Gets cluster diagnostic information. After the operation completes, the
188-
Operation.response field contains ``DiagnoseClusterOutputLocation``.
193+
Gets cluster diagnostic information. The returned ``Operation.metadata``
194+
will be
195+
`ClusterOperationMetadata <https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#clusteroperationmetadata>`__.
196+
After the operation completes, ``Operation.response`` contains
197+
`DiagnoseClusterResults <https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#diagnoseclusterresults>`__.
189198
190199
Returns:
191200
Callable: A callable which accepts the appropriate

‎dataproc/google/cloud/dataproc_v1/gapic/transports/workflow_template_service_grpc_transport.py‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -160,7 +160,10 @@ def instantiate_workflow_template(self):
160160
cause any inflight jobs to be cancelled and workflow-owned clusters to
161161
be deleted.
162162
163-
The ``Operation.metadata`` will be ``WorkflowMetadata``.
163+
The ``Operation.metadata`` will be
164+
`WorkflowMetadata <https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#workflowmetadata>`__.
165+
Also see `Using
166+
WorkflowMetadata <https://cloud.google.com/dataproc/docs/concepts/workflows/debugging#using_workflowmetadata>`__.
164167
165168
On successful completion, ``Operation.response`` will be ``Empty``.
166169
@@ -189,7 +192,10 @@ def instantiate_inline_workflow_template(self):
189192
cause any inflight jobs to be cancelled and workflow-owned clusters to
190193
be deleted.
191194
192-
The ``Operation.metadata`` will be ``WorkflowMetadata``.
195+
The ``Operation.metadata`` will be
196+
`WorkflowMetadata <https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#workflowmetadata>`__.
197+
Also see `Using
198+
WorkflowMetadata <https://cloud.google.com/dataproc/docs/concepts/workflows/debugging#using_workflowmetadata>`__.
193199
194200
On successful completion, ``Operation.response`` will be ``Empty``.
195201

‎dataproc/google/cloud/dataproc_v1/gapic/workflow_template_service_client.py‎

Lines changed: 69 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -241,9 +241,16 @@ def create_workflow_template(
241241
>>> response = client.create_workflow_template(parent, template)
242242
243243
Args:
244-
parent (str): Required. The "resource name" of the region, as described in
245-
https://cloud.google.com/apis/design/resource\_names of the form
246-
``projects/{project_id}/regions/{region}``
244+
parent (str): Required. The resource name of the region or location, as described in
245+
https://cloud.google.com/apis/design/resource\_names.
246+
247+
- For ``projects.regions.workflowTemplates,create``, the resource name
248+
of the region has the following format:
249+
``projects/{project_id}/regions/{region}``
250+
251+
- For ``projects.locations.workflowTemplates.create``, the resource
252+
name of the location has the following format:
253+
``projects/{project_id}/locations/{location}``
247254
template (Union[dict, ~google.cloud.dataproc_v1.types.WorkflowTemplate]): Required. The Dataproc workflow template to create.
248255
249256
If a dict is provided, it must be of the same form as the protobuf
@@ -322,11 +329,18 @@ def get_workflow_template(
322329
>>> response = client.get_workflow_template(name)
323330
324331
Args:
325-
name (str): Required. The "resource name" of the workflow template, as described in
326-
https://cloud.google.com/apis/design/resource\_names of the form
327-
``projects/{project_id}/regions/{region}/workflowTemplates/{template_id}``
332+
name (str): Required. The resource name of the workflow template, as described in
333+
https://cloud.google.com/apis/design/resource\_names.
334+
335+
- For ``projects.regions.workflowTemplates.get``, the resource name of
336+
the template has the following format:
337+
``projects/{project_id}/regions/{region}/workflowTemplates/{template_id}``
338+
339+
- For ``projects.locations.workflowTemplates.get``, the resource name
340+
of the template has the following format:
341+
``projects/{project_id}/locations/{location}/workflowTemplates/{template_id}``
328342
version (int): Optional. The version of workflow template to retrieve. Only previously
329-
instatiated versions can be retrieved.
343+
instantiated versions can be retrieved.
330344
331345
If unspecified, retrieves the current version.
332346
retry (Optional[google.api_core.retry.Retry]): A retry object used
@@ -400,7 +414,10 @@ def instantiate_workflow_template(
400414
cause any inflight jobs to be cancelled and workflow-owned clusters to
401415
be deleted.
402416
403-
The ``Operation.metadata`` will be ``WorkflowMetadata``.
417+
The ``Operation.metadata`` will be
418+
`WorkflowMetadata <https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#workflowmetadata>`__.
419+
Also see `Using
420+
WorkflowMetadata <https://cloud.google.com/dataproc/docs/concepts/workflows/debugging#using_workflowmetadata>`__.
404421
405422
On successful completion, ``Operation.response`` will be ``Empty``.
406423
@@ -423,9 +440,16 @@ def instantiate_workflow_template(
423440
>>> metadata = response.metadata()
424441
425442
Args:
426-
name (str): Required. The "resource name" of the workflow template, as described in
427-
https://cloud.google.com/apis/design/resource\_names of the form
428-
``projects/{project_id}/regions/{region}/workflowTemplates/{template_id}``
443+
name (str): Required. The resource name of the workflow template, as described in
444+
https://cloud.google.com/apis/design/resource\_names.
445+
446+
- For ``projects.regions.workflowTemplates.instantiate``, the resource
447+
name of the template has the following format:
448+
``projects/{project_id}/regions/{region}/workflowTemplates/{template_id}``
449+
450+
- For ``projects.locations.workflowTemplates.instantiate``, the
451+
resource name of the template has the following format:
452+
``projects/{project_id}/locations/{location}/workflowTemplates/{template_id}``
429453
version (int): Optional. The version of workflow template to instantiate. If specified,
430454
the workflow will be instantiated only if the current version of
431455
the workflow template has the supplied version.
@@ -525,7 +549,10 @@ def instantiate_inline_workflow_template(
525549
cause any inflight jobs to be cancelled and workflow-owned clusters to
526550
be deleted.
527551
528-
The ``Operation.metadata`` will be ``WorkflowMetadata``.
552+
The ``Operation.metadata`` will be
553+
`WorkflowMetadata <https://cloud.google.com/dataproc/docs/reference/rpc/google.cloud.dataproc.v1#workflowmetadata>`__.
554+
Also see `Using
555+
WorkflowMetadata <https://cloud.google.com/dataproc/docs/concepts/workflows/debugging#using_workflowmetadata>`__.
529556
530557
On successful completion, ``Operation.response`` will be ``Empty``.
531558
@@ -551,9 +578,16 @@ def instantiate_inline_workflow_template(
551578
>>> metadata = response.metadata()
552579
553580
Args:
554-
parent (str): Required. The "resource name" of the workflow template region, as
555-
described in https://cloud.google.com/apis/design/resource\_names of the
556-
form ``projects/{project_id}/regions/{region}``
581+
parent (str): Required. The resource name of the region or location, as described in
582+
https://cloud.google.com/apis/design/resource\_names.
583+
584+
- For ``projects.regions.workflowTemplates,instantiateinline``, the
585+
resource name of the region has the following format:
586+
``projects/{project_id}/regions/{region}``
587+
588+
- For ``projects.locations.workflowTemplates.instantiateinline``, the
589+
resource name of the location has the following format:
590+
``projects/{project_id}/locations/{location}``
557591
template (Union[dict, ~google.cloud.dataproc_v1.types.WorkflowTemplate]): Required. The workflow template to instantiate.
558592
559593
If a dict is provided, it must be of the same form as the protobuf
@@ -738,9 +772,16 @@ def list_workflow_templates(
738772
... pass
739773
740774
Args:
741-
parent (str): Required. The "resource name" of the region, as described in
742-
https://cloud.google.com/apis/design/resource\_names of the form
743-
``projects/{project_id}/regions/{region}``
775+
parent (str): Required. The resource name of the region or location, as described in
776+
https://cloud.google.com/apis/design/resource\_names.
777+
778+
- For ``projects.regions.workflowTemplates,list``, the resource name of
779+
the region has the following format:
780+
``projects/{project_id}/regions/{region}``
781+
782+
- For ``projects.locations.workflowTemplates.list``, the resource name
783+
of the location has the following format:
784+
``projects/{project_id}/locations/{location}``
744785
page_size (int): The maximum number of resources contained in the
745786
underlying API response. If page streaming is performed per-
746787
resource, this parameter does not affect the return value. If page
@@ -831,9 +872,16 @@ def delete_workflow_template(
831872
>>> client.delete_workflow_template(name)
832873
833874
Args:
834-
name (str): Required. The "resource name" of the workflow template, as described in
835-
https://cloud.google.com/apis/design/resource\_names of the form
836-
``projects/{project_id}/regions/{region}/workflowTemplates/{template_id}``
875+
name (str): Required. The resource name of the workflow template, as described in
876+
https://cloud.google.com/apis/design/resource\_names.
877+
878+
- For ``projects.regions.workflowTemplates.delete``, the resource name
879+
of the template has the following format:
880+
``projects/{project_id}/regions/{region}/workflowTemplates/{template_id}``
881+
882+
- For ``projects.locations.workflowTemplates.instantiate``, the
883+
resource name of the template has the following format:
884+
``projects/{project_id}/locations/{location}/workflowTemplates/{template_id}``
837885
version (int): Optional. The version of workflow template to delete. If specified,
838886
will only delete the template if the current server version matches
839887
specified version.

0 commit comments

Comments
 (0)