Skip to content
This repository was archived by the owner on Nov 29, 2023. It is now read-only.

Commit 1cf10b6

Browse files
yoshi-automationyoshi-automation
authored
feat: add SparkR and Presto jobs to WorkflowTemplates; add new optional components; add submit_job_as_operation to v1 (via synth) (#21)
1 parent ac2cc78 commit 1cf10b6

67 files changed

Lines changed: 4961 additions & 3485 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.kokoro/publish-docs.sh‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,6 @@
1313
# See the License for the specific language governing permissions and
1414
# limitations under the License.
1515

16-
#!/bin/bash
17-
1816
set -eo pipefail
1917

2018
# Disable buffering, so that the logs stream through.

‎docs/gapic/v1/api.rst‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
Client for Google Cloud Dataproc API
2-
====================================
1+
Client for Cloud Dataproc API
2+
=============================
33

44
.. automodule:: google.cloud.dataproc_v1
55
:members:

‎docs/gapic/v1/types.rst‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
Types for Google Cloud Dataproc API Client
2-
==========================================
1+
Types for Cloud Dataproc API Client
2+
===================================
33

44
.. automodule:: google.cloud.dataproc_v1.types
55
:members:

‎docs/gapic/v1beta2/api.rst‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
Client for Google Cloud Dataproc API
2-
====================================
1+
Client for Cloud Dataproc API
2+
=============================
33

44
.. automodule:: google.cloud.dataproc_v1beta2
55
:members:

‎docs/gapic/v1beta2/types.rst‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
Types for Google Cloud Dataproc API Client
2-
==========================================
1+
Types for Cloud Dataproc API Client
2+
===================================
33

44
.. automodule:: google.cloud.dataproc_v1beta2.types
55
:members:

‎docs/index.rst‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
.. include:: README.rst
22

3+
.. include:: multiprocessing.rst
4+
35
API Reference
46
-------------
57
.. toctree::

‎docs/multiprocessing.rst‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
.. note::
2+
3+
Because this client uses :mod:`grpcio` library, it is safe to
4+
share instances across threads. In multiprocessing scenarios, the best
5+
practice is to create client instances *after* the invocation of
6+
:func:`os.fork` by :class:`multiprocessing.Pool` or
7+
:class:`multiprocessing.Process`.

‎google/cloud/dataproc.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,8 @@
2828
__all__ = (
2929
"enums",
3030
"types",
31+
"AutoscalingPolicyServiceClient",
3132
"ClusterControllerClient",
3233
"JobControllerClient",
3334
"WorkflowTemplateServiceClient",
34-
"AutoscalingPolicyServiceClient",
3535
)

‎google/cloud/dataproc_v1/__init__.py‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,13 @@
3636
warnings.warn(message, DeprecationWarning)
3737

3838

39+
class AutoscalingPolicyServiceClient(
40+
autoscaling_policy_service_client.AutoscalingPolicyServiceClient
41+
):
42+
__doc__ = autoscaling_policy_service_client.AutoscalingPolicyServiceClient.__doc__
43+
enums = enums
44+
45+
3946
class ClusterControllerClient(cluster_controller_client.ClusterControllerClient):
4047
__doc__ = cluster_controller_client.ClusterControllerClient.__doc__
4148
enums = enums
@@ -53,18 +60,11 @@ class WorkflowTemplateServiceClient(
5360
enums = enums
5461

5562

56-
class AutoscalingPolicyServiceClient(
57-
autoscaling_policy_service_client.AutoscalingPolicyServiceClient
58-
):
59-
__doc__ = autoscaling_policy_service_client.AutoscalingPolicyServiceClient.__doc__
60-
enums = enums
61-
62-
6363
__all__ = (
6464
"enums",
6565
"types",
66+
"AutoscalingPolicyServiceClient",
6667
"ClusterControllerClient",
6768
"JobControllerClient",
6869
"WorkflowTemplateServiceClient",
69-
"AutoscalingPolicyServiceClient",
7070
)

‎google/cloud/dataproc_v1/gapic/autoscaling_policy_service_client.py‎

Lines changed: 64 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -38,17 +38,7 @@
3838
)
3939
from google.cloud.dataproc_v1.proto import autoscaling_policies_pb2
4040
from google.cloud.dataproc_v1.proto import autoscaling_policies_pb2_grpc
41-
from google.cloud.dataproc_v1.proto import clusters_pb2
42-
from google.cloud.dataproc_v1.proto import clusters_pb2_grpc
43-
from google.cloud.dataproc_v1.proto import jobs_pb2
44-
from google.cloud.dataproc_v1.proto import jobs_pb2_grpc
45-
from google.cloud.dataproc_v1.proto import operations_pb2 as proto_operations_pb2
46-
from google.cloud.dataproc_v1.proto import workflow_templates_pb2
47-
from google.cloud.dataproc_v1.proto import workflow_templates_pb2_grpc
48-
from google.longrunning import operations_pb2 as longrunning_operations_pb2
49-
from google.protobuf import duration_pb2
5041
from google.protobuf import empty_pb2
51-
from google.protobuf import field_mask_pb2
5242

5343

5444
_GAPIC_LIBRARY_VERSION = pkg_resources.get_distribution("google-cloud-dataproc").version
@@ -88,15 +78,24 @@ def from_service_account_file(cls, filename, *args, **kwargs):
8878
from_service_account_json = from_service_account_file
8979

9080
@classmethod
91-
def autoscaling_policy_path(cls, project, region, autoscaling_policy):
81+
def autoscaling_policy_path(cls, project, location, autoscaling_policy):
9282
"""Return a fully-qualified autoscaling_policy string."""
9383
return google.api_core.path_template.expand(
94-
"projects/{project}/regions/{region}/autoscalingPolicies/{autoscaling_policy}",
84+
"projects/{project}/locations/{location}/autoscalingPolicies/{autoscaling_policy}",
9585
project=project,
96-
region=region,
86+
location=location,
9787
autoscaling_policy=autoscaling_policy,
9888
)
9989

90+
@classmethod
91+
def location_path(cls, project, location):
92+
"""Return a fully-qualified location string."""
93+
return google.api_core.path_template.expand(
94+
"projects/{project}/locations/{location}",
95+
project=project,
96+
location=location,
97+
)
98+
10099
@classmethod
101100
def region_path(cls, project, region):
102101
"""Return a fully-qualified region string."""
@@ -217,41 +216,31 @@ def __init__(
217216
self._inner_api_calls = {}
218217

219218
# Service calls
220-
def create_autoscaling_policy(
219+
def update_autoscaling_policy(
221220
self,
222-
parent,
223221
policy,
224222
retry=google.api_core.gapic_v1.method.DEFAULT,
225223
timeout=google.api_core.gapic_v1.method.DEFAULT,
226224
metadata=None,
227225
):
228226
"""
229-
Creates new autoscaling policy.
227+
Updates (replaces) autoscaling policy.
228+
229+
Disabled check for update_mask, because all updates will be full
230+
replacements.
230231
231232
Example:
232233
>>> from google.cloud import dataproc_v1
233234
>>>
234235
>>> client = dataproc_v1.AutoscalingPolicyServiceClient()
235236
>>>
236-
>>> parent = client.region_path('[PROJECT]', '[REGION]')
237-
>>>
238237
>>> # TODO: Initialize `policy`:
239238
>>> policy = {}
240239
>>>
241-
>>> response = client.create_autoscaling_policy(parent, policy)
240+
>>> response = client.update_autoscaling_policy(policy)
242241
243242
Args:
244-
parent (str): Required. The "resource name" of the region or location, as described in
245-
https://cloud.google.com/apis/design/resource\_names.
246-
247-
- For ``projects.regions.autoscalingPolicies.create``, the resource
248-
name of the region has the following format:
249-
``projects/{project_id}/regions/{region}``
250-
251-
- For ``projects.locations.autoscalingPolicies.create``, the resource
252-
name of the location has the following format:
253-
``projects/{project_id}/locations/{location}``
254-
policy (Union[dict, ~google.cloud.dataproc_v1.types.AutoscalingPolicy]): The autoscaling policy to create.
243+
policy (Union[dict, ~google.cloud.dataproc_v1.types.AutoscalingPolicy]): Required. The updated autoscaling policy.
255244
256245
If a dict is provided, it must be of the same form as the protobuf
257246
message :class:`~google.cloud.dataproc_v1.types.AutoscalingPolicy`
@@ -275,24 +264,22 @@ def create_autoscaling_policy(
275264
ValueError: If the parameters are invalid.
276265
"""
277266
# Wrap the transport method to add retry and timeout logic.
278-
if "create_autoscaling_policy" not in self._inner_api_calls:
267+
if "update_autoscaling_policy" not in self._inner_api_calls:
279268
self._inner_api_calls[
280-
"create_autoscaling_policy"
269+
"update_autoscaling_policy"
281270
] = google.api_core.gapic_v1.method.wrap_method(
282-
self.transport.create_autoscaling_policy,
283-
default_retry=self._method_configs["CreateAutoscalingPolicy"].retry,
284-
default_timeout=self._method_configs["CreateAutoscalingPolicy"].timeout,
271+
self.transport.update_autoscaling_policy,
272+
default_retry=self._method_configs["UpdateAutoscalingPolicy"].retry,
273+
default_timeout=self._method_configs["UpdateAutoscalingPolicy"].timeout,
285274
client_info=self._client_info,
286275
)
287276

288-
request = autoscaling_policies_pb2.CreateAutoscalingPolicyRequest(
289-
parent=parent, policy=policy
290-
)
277+
request = autoscaling_policies_pb2.UpdateAutoscalingPolicyRequest(policy=policy)
291278
if metadata is None:
292279
metadata = []
293280
metadata = list(metadata)
294281
try:
295-
routing_header = [("parent", parent)]
282+
routing_header = [("policy.name", policy.name)]
296283
except AttributeError:
297284
pass
298285
else:
@@ -301,35 +288,45 @@ def create_autoscaling_policy(
301288
)
302289
metadata.append(routing_metadata)
303290

304-
return self._inner_api_calls["create_autoscaling_policy"](
291+
return self._inner_api_calls["update_autoscaling_policy"](
305292
request, retry=retry, timeout=timeout, metadata=metadata
306293
)
307294

308-
def update_autoscaling_policy(
295+
def create_autoscaling_policy(
309296
self,
297+
parent,
310298
policy,
311299
retry=google.api_core.gapic_v1.method.DEFAULT,
312300
timeout=google.api_core.gapic_v1.method.DEFAULT,
313301
metadata=None,
314302
):
315303
"""
316-
Updates (replaces) autoscaling policy.
317-
318-
Disabled check for update\_mask, because all updates will be full
319-
replacements.
304+
Creates new autoscaling policy.
320305
321306
Example:
322307
>>> from google.cloud import dataproc_v1
323308
>>>
324309
>>> client = dataproc_v1.AutoscalingPolicyServiceClient()
325310
>>>
311+
>>> parent = client.region_path('[PROJECT]', '[REGION]')
312+
>>>
326313
>>> # TODO: Initialize `policy`:
327314
>>> policy = {}
328315
>>>
329-
>>> response = client.update_autoscaling_policy(policy)
316+
>>> response = client.create_autoscaling_policy(parent, policy)
330317
331318
Args:
332-
policy (Union[dict, ~google.cloud.dataproc_v1.types.AutoscalingPolicy]): Required. The updated autoscaling policy.
319+
parent (str): Required. The "resource name" of the region or location, as
320+
described in https://cloud.google.com/apis/design/resource_names.
321+
322+
- For ``projects.regions.autoscalingPolicies.create``, the resource
323+
name of the region has the following format:
324+
``projects/{project_id}/regions/{region}``
325+
326+
- For ``projects.locations.autoscalingPolicies.create``, the resource
327+
name of the location has the following format:
328+
``projects/{project_id}/locations/{location}``
329+
policy (Union[dict, ~google.cloud.dataproc_v1.types.AutoscalingPolicy]): Required. The autoscaling policy to create.
333330
334331
If a dict is provided, it must be of the same form as the protobuf
335332
message :class:`~google.cloud.dataproc_v1.types.AutoscalingPolicy`
@@ -353,22 +350,24 @@ def update_autoscaling_policy(
353350
ValueError: If the parameters are invalid.
354351
"""
355352
# Wrap the transport method to add retry and timeout logic.
356-
if "update_autoscaling_policy" not in self._inner_api_calls:
353+
if "create_autoscaling_policy" not in self._inner_api_calls:
357354
self._inner_api_calls[
358-
"update_autoscaling_policy"
355+
"create_autoscaling_policy"
359356
] = google.api_core.gapic_v1.method.wrap_method(
360-
self.transport.update_autoscaling_policy,
361-
default_retry=self._method_configs["UpdateAutoscalingPolicy"].retry,
362-
default_timeout=self._method_configs["UpdateAutoscalingPolicy"].timeout,
357+
self.transport.create_autoscaling_policy,
358+
default_retry=self._method_configs["CreateAutoscalingPolicy"].retry,
359+
default_timeout=self._method_configs["CreateAutoscalingPolicy"].timeout,
363360
client_info=self._client_info,
364361
)
365362

366-
request = autoscaling_policies_pb2.UpdateAutoscalingPolicyRequest(policy=policy)
363+
request = autoscaling_policies_pb2.CreateAutoscalingPolicyRequest(
364+
parent=parent, policy=policy
365+
)
367366
if metadata is None:
368367
metadata = []
369368
metadata = list(metadata)
370369
try:
371-
routing_header = [("policy.name", policy.name)]
370+
routing_header = [("parent", parent)]
372371
except AttributeError:
373372
pass
374373
else:
@@ -377,7 +376,7 @@ def update_autoscaling_policy(
377376
)
378377
metadata.append(routing_metadata)
379378

380-
return self._inner_api_calls["update_autoscaling_policy"](
379+
return self._inner_api_calls["create_autoscaling_policy"](
381380
request, retry=retry, timeout=timeout, metadata=metadata
382381
)
383382

@@ -396,13 +395,14 @@ def get_autoscaling_policy(
396395
>>>
397396
>>> client = dataproc_v1.AutoscalingPolicyServiceClient()
398397
>>>
399-
>>> name = client.autoscaling_policy_path('[PROJECT]', '[REGION]', '[AUTOSCALING_POLICY]')
398+
>>> # TODO: Initialize `name`:
399+
>>> name = ''
400400
>>>
401401
>>> response = client.get_autoscaling_policy(name)
402402
403403
Args:
404-
name (str): Required. The "resource name" of the autoscaling policy, as described in
405-
https://cloud.google.com/apis/design/resource\_names.
404+
name (str): Required. The "resource name" of the autoscaling policy, as
405+
described in https://cloud.google.com/apis/design/resource_names.
406406
407407
- For ``projects.regions.autoscalingPolicies.get``, the resource name
408408
of the policy has the following format:
@@ -492,8 +492,8 @@ def list_autoscaling_policies(
492492
... pass
493493
494494
Args:
495-
parent (str): Required. The "resource name" of the region or location, as described in
496-
https://cloud.google.com/apis/design/resource\_names.
495+
parent (str): Required. The "resource name" of the region or location, as
496+
described in https://cloud.google.com/apis/design/resource_names.
497497
498498
- For ``projects.regions.autoscalingPolicies.list``, the resource name
499499
of the region has the following format:
@@ -587,13 +587,14 @@ def delete_autoscaling_policy(
587587
>>>
588588
>>> client = dataproc_v1.AutoscalingPolicyServiceClient()
589589
>>>
590-
>>> name = client.autoscaling_policy_path('[PROJECT]', '[REGION]', '[AUTOSCALING_POLICY]')
590+
>>> # TODO: Initialize `name`:
591+
>>> name = ''
591592
>>>
592593
>>> client.delete_autoscaling_policy(name)
593594
594595
Args:
595-
name (str): Required. The "resource name" of the autoscaling policy, as described in
596-
https://cloud.google.com/apis/design/resource\_names.
596+
name (str): Required. The "resource name" of the autoscaling policy, as
597+
described in https://cloud.google.com/apis/design/resource_names.
597598
598599
- For ``projects.regions.autoscalingPolicies.delete``, the resource
599600
name of the policy has the following format:

0 commit comments

Comments
 (0)