diff --git a/.release-please-manifest.json b/.release-please-manifest.json index b5e91573..ce65b73e 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "5.2.0" + ".": "5.3.0" } diff --git a/CHANGELOG.md b/CHANGELOG.md index d6d0d5e7..3641a2ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,23 @@ [1]: https://pypi.org/project/google-cloud-dataproc/#history +## [5.3.0](https://github.com/googleapis/python-dataproc/compare/v5.2.0...v5.3.0) (2023-01-23) + + +### Features + +* Add SPOT to Preemptibility enum ([8d5e6d8](https://github.com/googleapis/python-dataproc/commit/8d5e6d8b756bffa44227bdf5dd27223e45facd57)) + + +### Bug Fixes + +* Add context manager return types ([8d5e6d8](https://github.com/googleapis/python-dataproc/commit/8d5e6d8b756bffa44227bdf5dd27223e45facd57)) + + +### Documentation + +* Add documentation for enums ([8d5e6d8](https://github.com/googleapis/python-dataproc/commit/8d5e6d8b756bffa44227bdf5dd27223e45facd57)) + ## [5.2.0](https://github.com/googleapis/python-dataproc/compare/v5.1.0...v5.2.0) (2023-01-10) diff --git a/google/cloud/dataproc/gapic_version.py b/google/cloud/dataproc/gapic_version.py index 72f20eb7..76351bda 100644 --- a/google/cloud/dataproc/gapic_version.py +++ b/google/cloud/dataproc/gapic_version.py @@ -13,4 +13,4 @@ # See the License for the specific language governing permissions and # limitations under the License. # -__version__ = "5.2.0" # {x-release-please-version} +__version__ = "5.3.0" # {x-release-please-version} diff --git a/google/cloud/dataproc_v1/gapic_version.py b/google/cloud/dataproc_v1/gapic_version.py index 72f20eb7..76351bda 100644 --- a/google/cloud/dataproc_v1/gapic_version.py +++ b/google/cloud/dataproc_v1/gapic_version.py @@ -13,4 +13,4 @@ # See the License for the specific language governing permissions and # limitations under the License. # -__version__ = "5.2.0" # {x-release-please-version} +__version__ = "5.3.0" # {x-release-please-version} diff --git a/google/cloud/dataproc_v1/services/autoscaling_policy_service/client.py b/google/cloud/dataproc_v1/services/autoscaling_policy_service/client.py index 2cb6625d..0a434a38 100644 --- a/google/cloud/dataproc_v1/services/autoscaling_policy_service/client.py +++ b/google/cloud/dataproc_v1/services/autoscaling_policy_service/client.py @@ -1044,7 +1044,7 @@ def sample_delete_autoscaling_policy(): metadata=metadata, ) - def __enter__(self): + def __enter__(self) -> "AutoscalingPolicyServiceClient": return self def __exit__(self, type, value, traceback): diff --git a/google/cloud/dataproc_v1/services/batch_controller/client.py b/google/cloud/dataproc_v1/services/batch_controller/client.py index a96592e9..05d0e9f0 100644 --- a/google/cloud/dataproc_v1/services/batch_controller/client.py +++ b/google/cloud/dataproc_v1/services/batch_controller/client.py @@ -898,7 +898,7 @@ def sample_delete_batch(): metadata=metadata, ) - def __enter__(self): + def __enter__(self) -> "BatchControllerClient": return self def __exit__(self, type, value, traceback): diff --git a/google/cloud/dataproc_v1/services/cluster_controller/client.py b/google/cloud/dataproc_v1/services/cluster_controller/client.py index f52ecb3b..921a308c 100644 --- a/google/cloud/dataproc_v1/services/cluster_controller/client.py +++ b/google/cloud/dataproc_v1/services/cluster_controller/client.py @@ -1654,7 +1654,7 @@ def sample_diagnose_cluster(): # Done; return the response. return response - def __enter__(self): + def __enter__(self) -> "ClusterControllerClient": return self def __exit__(self, type, value, traceback): diff --git a/google/cloud/dataproc_v1/services/job_controller/client.py b/google/cloud/dataproc_v1/services/job_controller/client.py index 810299d9..a5d75602 100644 --- a/google/cloud/dataproc_v1/services/job_controller/client.py +++ b/google/cloud/dataproc_v1/services/job_controller/client.py @@ -1314,7 +1314,7 @@ def sample_delete_job(): metadata=metadata, ) - def __enter__(self): + def __enter__(self) -> "JobControllerClient": return self def __exit__(self, type, value, traceback): diff --git a/google/cloud/dataproc_v1/services/node_group_controller/client.py b/google/cloud/dataproc_v1/services/node_group_controller/client.py index af9932ae..402b31e8 100644 --- a/google/cloud/dataproc_v1/services/node_group_controller/client.py +++ b/google/cloud/dataproc_v1/services/node_group_controller/client.py @@ -835,7 +835,7 @@ def sample_get_node_group(): # Done; return the response. return response - def __enter__(self): + def __enter__(self) -> "NodeGroupControllerClient": return self def __exit__(self, type, value, traceback): diff --git a/google/cloud/dataproc_v1/services/workflow_template_service/client.py b/google/cloud/dataproc_v1/services/workflow_template_service/client.py index 7241df04..c13dacf5 100644 --- a/google/cloud/dataproc_v1/services/workflow_template_service/client.py +++ b/google/cloud/dataproc_v1/services/workflow_template_service/client.py @@ -1452,7 +1452,7 @@ def sample_delete_workflow_template(): metadata=metadata, ) - def __enter__(self): + def __enter__(self) -> "WorkflowTemplateServiceClient": return self def __exit__(self, type, value, traceback): diff --git a/google/cloud/dataproc_v1/types/batches.py b/google/cloud/dataproc_v1/types/batches.py index 1aa48877..ee6ac168 100644 --- a/google/cloud/dataproc_v1/types/batches.py +++ b/google/cloud/dataproc_v1/types/batches.py @@ -250,7 +250,25 @@ class Batch(proto.Message): """ class State(proto.Enum): - r"""The batch state.""" + r"""The batch state. + + Values: + STATE_UNSPECIFIED (0): + The batch state is unknown. + PENDING (1): + The batch is created before running. + RUNNING (2): + The batch is running. + CANCELLING (3): + The batch is cancelling. + CANCELLED (4): + The batch cancellation was successful. + SUCCEEDED (5): + The batch completed successfully. + FAILED (6): + The batch is no longer running due to an + error. + """ STATE_UNSPECIFIED = 0 PENDING = 1 RUNNING = 2 diff --git a/google/cloud/dataproc_v1/types/clusters.py b/google/cloud/dataproc_v1/types/clusters.py index 070a255e..c761bce5 100644 --- a/google/cloud/dataproc_v1/types/clusters.py +++ b/google/cloud/dataproc_v1/types/clusters.py @@ -577,6 +577,24 @@ class PrivateIpv6GoogleAccess(proto.Enum): IPv6. These values are directly mapped to corresponding values in the `Compute Engine Instance fields `__. + + Values: + PRIVATE_IPV6_GOOGLE_ACCESS_UNSPECIFIED (0): + If unspecified, Compute Engine default behavior will apply, + which is the same as + [INHERIT_FROM_SUBNETWORK][google.cloud.dataproc.v1.GceClusterConfig.PrivateIpv6GoogleAccess.INHERIT_FROM_SUBNETWORK]. + INHERIT_FROM_SUBNETWORK (1): + Private access to and from Google Services + configuration inherited from the subnetwork + configuration. This is the default Compute + Engine behavior. + OUTBOUND (2): + Enables outbound private IPv6 access to + Google Services from the Dataproc cluster. + BIDIRECTIONAL (3): + Enables bidirectional private IPv6 access + between Google Services and the Dataproc + cluster. """ PRIVATE_IPV6_GOOGLE_ACCESS_UNSPECIFIED = 0 INHERIT_FROM_SUBNETWORK = 1 @@ -795,10 +813,39 @@ class InstanceGroupConfig(proto.Message): """ class Preemptibility(proto.Enum): - r"""Controls the use of preemptible instances within the group.""" + r"""Controls the use of preemptible instances within the group. + + Values: + PREEMPTIBILITY_UNSPECIFIED (0): + Preemptibility is unspecified, the system + will choose the appropriate setting for each + instance group. + NON_PREEMPTIBLE (1): + Instances are non-preemptible. + This option is allowed for all instance groups + and is the only valid value for Master and + Worker instance groups. + PREEMPTIBLE (2): + Instances are [preemptible] + (https://cloud.google.com/compute/docs/instances/preemptible). + + This option is allowed only for [secondary worker] + (https://cloud.google.com/dataproc/docs/concepts/compute/secondary-vms) + groups. + SPOT (3): + Instances are [Spot VMs] + (https://cloud.google.com/compute/docs/instances/spot). + + This option is allowed only for [secondary worker] + (https://cloud.google.com/dataproc/docs/concepts/compute/secondary-vms) + groups. Spot VMs are the latest version of [preemptible VMs] + (https://cloud.google.com/compute/docs/instances/preemptible), + and provide additional features. + """ PREEMPTIBILITY_UNSPECIFIED = 0 NON_PREEMPTIBLE = 1 PREEMPTIBLE = 2 + SPOT = 3 num_instances: int = proto.Field( proto.INT32, @@ -1007,7 +1054,14 @@ class NodeGroup(proto.Message): """ class Role(proto.Enum): - r"""Node group roles.""" + r"""Node group roles. + + Values: + ROLE_UNSPECIFIED (0): + Required unspecified role. + DRIVER (1): + Job drivers run on the node group. + """ ROLE_UNSPECIFIED = 0 DRIVER = 1 @@ -1081,7 +1135,45 @@ class ClusterStatus(proto.Message): """ class State(proto.Enum): - r"""The cluster state.""" + r"""The cluster state. + + Values: + UNKNOWN (0): + The cluster state is unknown. + CREATING (1): + The cluster is being created and set up. It + is not ready for use. + RUNNING (2): + The cluster is currently running and healthy. It is ready + for use. + + **Note:** The cluster state changes from "creating" to + "running" status after the master node(s), first two primary + worker nodes (and the last primary worker node if primary + workers > 2) are running. + ERROR (3): + The cluster encountered an error. It is not + ready for use. + ERROR_DUE_TO_UPDATE (9): + The cluster has encountered an error while + being updated. Jobs can be submitted to the + cluster, but the cluster cannot be updated. + DELETING (4): + The cluster is being deleted. It cannot be + used. + UPDATING (5): + The cluster is being updated. It continues to + accept and process jobs. + STOPPING (6): + The cluster is being stopped. It cannot be + used. + STOPPED (7): + The cluster is currently stopped. It is not + ready for use. + STARTING (8): + The cluster is being started. It is not ready + for use. + """ UNKNOWN = 0 CREATING = 1 RUNNING = 2 @@ -1094,7 +1186,23 @@ class State(proto.Enum): STARTING = 8 class Substate(proto.Enum): - r"""The cluster substate.""" + r"""The cluster substate. + + Values: + UNSPECIFIED (0): + The cluster substate is unknown. + UNHEALTHY (1): + The cluster is known to be in an unhealthy + state (for example, critical daemons are not + running or HDFS capacity is exhausted). + + Applies to RUNNING state. + STALE_STATUS (2): + The agent-reported status is out of date (may + occur if Dataproc loses communication with + Agent). + Applies to RUNNING state. + """ UNSPECIFIED = 0 UNHEALTHY = 1 STALE_STATUS = 2 @@ -1445,6 +1553,25 @@ class MetricSource(proto.Enum): r"""A source for the collection of Dataproc OSS metrics (see [available OSS metrics] (https://cloud.google.com//dataproc/docs/guides/monitoring#available_oss_metrics)). + + Values: + METRIC_SOURCE_UNSPECIFIED (0): + Required unspecified metric source. + MONITORING_AGENT_DEFAULTS (1): + Default monitoring agent metrics. If this source is enabled, + Dataproc enables the monitoring agent in Compute Engine, and + collects default monitoring agent metrics, which are + published with an ``agent.googleapis.com`` prefix. + HDFS (2): + HDFS metric source. + SPARK (3): + Spark metric source. + YARN (4): + YARN metric source. + SPARK_HISTORY_SERVER (5): + Spark History Server metric source. + HIVESERVER2 (6): + Hiveserver2 metric source. """ METRIC_SOURCE_UNSPECIFIED = 0 MONITORING_AGENT_DEFAULTS = 1 @@ -2077,6 +2204,18 @@ class ReservationAffinity(proto.Message): class Type(proto.Enum): r"""Indicates whether to consume capacity from an reservation or not. + + Values: + TYPE_UNSPECIFIED (0): + + NO_RESERVATION (1): + Do not consume from any allocated capacity. + ANY_RESERVATION (2): + Consume any reservation available. + SPECIFIC_RESERVATION (3): + Must consume from a specific reservation. + Must specify key value fields for specifying the + reservations. """ TYPE_UNSPECIFIED = 0 NO_RESERVATION = 1 diff --git a/google/cloud/dataproc_v1/types/jobs.py b/google/cloud/dataproc_v1/types/jobs.py index 501b9446..0f9f70f2 100644 --- a/google/cloud/dataproc_v1/types/jobs.py +++ b/google/cloud/dataproc_v1/types/jobs.py @@ -69,6 +69,27 @@ class Level(proto.Enum): r"""The Log4j level for job execution. When running an `Apache Hive `__ job, Cloud Dataproc configures the Hive client to an equivalent verbosity level. + + Values: + LEVEL_UNSPECIFIED (0): + Level is unspecified. Use default level for + log4j. + ALL (1): + Use ALL level for log4j. + TRACE (2): + Use TRACE level for log4j. + DEBUG (3): + Use DEBUG level for log4j. + INFO (4): + Use INFO level for log4j. + WARN (5): + Use WARN level for log4j. + ERROR (6): + Use ERROR level for log4j. + FATAL (7): + Use FATAL level for log4j. + OFF (8): + Turn off log4j. """ LEVEL_UNSPECIFIED = 0 ALL = 1 @@ -797,7 +818,40 @@ class JobStatus(proto.Message): """ class State(proto.Enum): - r"""The job state.""" + r"""The job state. + + Values: + STATE_UNSPECIFIED (0): + The job state is unknown. + PENDING (1): + The job is pending; it has been submitted, + but is not yet running. + SETUP_DONE (8): + Job has been received by the service and + completed initial setup; it will soon be + submitted to the cluster. + RUNNING (2): + The job is running on the cluster. + CANCEL_PENDING (3): + A CancelJob request has been received, but is + pending. + CANCEL_STARTED (7): + Transient in-flight resources have been + canceled, and the request to cancel the running + job has been issued to the cluster. + CANCELLED (4): + The job cancellation was successful. + DONE (5): + The job has completed successfully. + ERROR (6): + The job has completed, but encountered an + error. + ATTEMPT_FAILURE (9): + Job attempt has failed. The detail field + contains failure details for this attempt. + + Applies to restartable jobs only. + """ STATE_UNSPECIFIED = 0 PENDING = 1 SETUP_DONE = 8 @@ -810,7 +864,29 @@ class State(proto.Enum): ATTEMPT_FAILURE = 9 class Substate(proto.Enum): - r"""The job substate.""" + r"""The job substate. + + Values: + UNSPECIFIED (0): + The job substate is unknown. + SUBMITTED (1): + The Job is submitted to the agent. + Applies to RUNNING state. + QUEUED (2): + The Job has been received and is awaiting + execution (it may be waiting for a condition to + be met). See the "details" field for the reason + for the delay. + + Applies to RUNNING state. + STALE_STATUS (3): + The agent-reported status is out of date, + which may be caused by a loss of communication + between the agent and Dataproc. If the agent + does not send a timely update, the job will + fail. + Applies to RUNNING state. + """ UNSPECIFIED = 0 SUBMITTED = 1 QUEUED = 2 @@ -895,6 +971,26 @@ class YarnApplication(proto.Message): class State(proto.Enum): r"""The application state, corresponding to YarnProtos.YarnApplicationStateProto. + + Values: + STATE_UNSPECIFIED (0): + Status is unspecified. + NEW (1): + Status is NEW. + NEW_SAVING (2): + Status is NEW_SAVING. + SUBMITTED (3): + Status is SUBMITTED. + ACCEPTED (4): + Status is ACCEPTED. + RUNNING (5): + Status is RUNNING. + FINISHED (6): + Status is FINISHED. + FAILED (7): + Status is FAILED. + KILLED (8): + Status is KILLED. """ STATE_UNSPECIFIED = 0 NEW = 1 @@ -1344,7 +1440,18 @@ class ListJobsRequest(proto.Message): """ class JobStateMatcher(proto.Enum): - r"""A matcher that specifies categories of job states.""" + r"""A matcher that specifies categories of job states. + + Values: + ALL (0): + Match all jobs, regardless of state. + ACTIVE (1): + Only match jobs in non-terminal states: PENDING, RUNNING, or + CANCEL_PENDING. + NON_ACTIVE (2): + Only match jobs in terminal states: + CANCELLED, DONE, or ERROR. + """ ALL = 0 ACTIVE = 1 NON_ACTIVE = 2 diff --git a/google/cloud/dataproc_v1/types/operations.py b/google/cloud/dataproc_v1/types/operations.py index 3b7e353d..e6c3c60f 100644 --- a/google/cloud/dataproc_v1/types/operations.py +++ b/google/cloud/dataproc_v1/types/operations.py @@ -55,7 +55,14 @@ class BatchOperationMetadata(proto.Message): """ class BatchOperationType(proto.Enum): - r"""Operation type for Batch resources""" + r"""Operation type for Batch resources + + Values: + BATCH_OPERATION_TYPE_UNSPECIFIED (0): + Batch operation type is unknown. + BATCH (1): + Batch operation type. + """ BATCH_OPERATION_TYPE_UNSPECIFIED = 0 BATCH = 1 @@ -115,7 +122,19 @@ class ClusterOperationStatus(proto.Message): """ class State(proto.Enum): - r"""The operation state.""" + r"""The operation state. + + Values: + UNKNOWN (0): + Unused. + PENDING (1): + The operation has been created. + RUNNING (2): + The operation is running. + DONE (3): + The operation is done; either cancelled or + completed. + """ UNKNOWN = 0 PENDING = 1 RUNNING = 2 @@ -229,7 +248,20 @@ class NodeGroupOperationMetadata(proto.Message): """ class NodeGroupOperationType(proto.Enum): - r"""Operation type for node group resources.""" + r"""Operation type for node group resources. + + Values: + NODE_GROUP_OPERATION_TYPE_UNSPECIFIED (0): + Node group operation type is unknown. + CREATE (1): + Create node group operation type. + UPDATE (2): + Update node group operation type. + DELETE (3): + Delete node group operation type. + RESIZE (4): + Resize node group operation type. + """ NODE_GROUP_OPERATION_TYPE_UNSPECIFIED = 0 CREATE = 1 UPDATE = 2 diff --git a/google/cloud/dataproc_v1/types/shared.py b/google/cloud/dataproc_v1/types/shared.py index 95603a13..5b3ba322 100644 --- a/google/cloud/dataproc_v1/types/shared.py +++ b/google/cloud/dataproc_v1/types/shared.py @@ -39,7 +39,43 @@ class Component(proto.Enum): - r"""Cluster components that can be activated.""" + r"""Cluster components that can be activated. + + Values: + COMPONENT_UNSPECIFIED (0): + Unspecified component. Specifying this will + cause Cluster creation to fail. + ANACONDA (5): + The Anaconda python distribution. The + Anaconda component is not supported in the + Dataproc 2.0 + image. The 2.0 image is pre-installed with + Miniconda. + DOCKER (13): + Docker + DRUID (9): + The Druid query engine. (alpha) + FLINK (14): + Flink + HBASE (11): + HBase. (beta) + HIVE_WEBHCAT (3): + The Hive Web HCatalog (the REST service for + accessing HCatalog). + JUPYTER (1): + The Jupyter Notebook. + PRESTO (6): + The Presto query engine. + RANGER (12): + The Ranger service. + SOLR (10): + The Solr service. + ZEPPELIN (4): + The Zeppelin notebook. + ZOOKEEPER (8): + The Zookeeper service. + """ COMPONENT_UNSPECIFIED = 0 ANACONDA = 5 DOCKER = 13 @@ -58,6 +94,16 @@ class Component(proto.Enum): class FailureAction(proto.Enum): r"""Actions in response to failure of a resource associated with a cluster. + + Values: + FAILURE_ACTION_UNSPECIFIED (0): + When FailureAction is unspecified, failure action defaults + to NO_ACTION. + NO_ACTION (1): + Take no action on failure to create a cluster resource. + NO_ACTION is the default. + DELETE (2): + Delete the failed cluster resource. """ FAILURE_ACTION_UNSPECIFIED = 0 NO_ACTION = 1 @@ -389,6 +435,19 @@ class Role(proto.Enum): can be specific to workloads. Exactly one GkeNodePoolTarget within the VirtualCluster must have 'default' role, which is used to run all workloads that are not associated with a NodePool. + + Values: + ROLE_UNSPECIFIED (0): + Role is unspecified. + DEFAULT (1): + Any roles that are not directly assigned to a NodePool run + on the ``default`` role's NodePool. + CONTROLLER (2): + Run controllers and webhooks. + SPARK_DRIVER (3): + Run spark driver. + SPARK_EXECUTOR (4): + Run spark executors. """ ROLE_UNSPECIFIED = 0 DEFAULT = 1 diff --git a/google/cloud/dataproc_v1/types/workflow_templates.py b/google/cloud/dataproc_v1/types/workflow_templates.py index 08ec9eda..d1b28de0 100644 --- a/google/cloud/dataproc_v1/types/workflow_templates.py +++ b/google/cloud/dataproc_v1/types/workflow_templates.py @@ -652,7 +652,19 @@ class WorkflowMetadata(proto.Message): """ class State(proto.Enum): - r"""The operation state.""" + r"""The operation state. + + Values: + UNKNOWN (0): + Unused. + PENDING (1): + The operation has been created. + RUNNING (2): + The operation is running. + DONE (3): + The operation is done; either cancelled or + completed. + """ UNKNOWN = 0 PENDING = 1 RUNNING = 2 @@ -785,7 +797,24 @@ class WorkflowNode(proto.Message): """ class NodeState(proto.Enum): - r"""The workflow node state.""" + r"""The workflow node state. + + Values: + NODE_STATE_UNSPECIFIED (0): + State is unspecified. + BLOCKED (1): + The node is awaiting prerequisite node to + finish. + RUNNABLE (2): + The node is runnable but not running. + RUNNING (3): + The node is running. + COMPLETED (4): + The node completed successfully. + FAILED (5): + The node failed. A node can be marked FAILED + because its ancestor or peer failed. + """ NODE_STATE_UNSPECIFIED = 0 BLOCKED = 1 RUNNABLE = 2 diff --git a/samples/generated_samples/snippet_metadata_google.cloud.dataproc.v1.json b/samples/generated_samples/snippet_metadata_google.cloud.dataproc.v1.json index 6638dc7d..121dcc1c 100644 --- a/samples/generated_samples/snippet_metadata_google.cloud.dataproc.v1.json +++ b/samples/generated_samples/snippet_metadata_google.cloud.dataproc.v1.json @@ -8,7 +8,7 @@ ], "language": "PYTHON", "name": "google-cloud-dataproc", - "version": "5.2.0" + "version": "5.3.0" }, "snippets": [ {