From 376b4dc2115baf84669f02186b8b14f35be382de Mon Sep 17 00:00:00 2001 From: Jesse Lovelace Date: Wed, 28 Apr 2021 10:22:01 -0700 Subject: [PATCH 1/3] Add retry sample to STS --- .../get_transfer_job_with_retries.py | 41 +++++++++++++++++++ storage/transfer_service/sts_snippets_test.py | 37 +++++++++++++++++ 2 files changed, 78 insertions(+) create mode 100644 storage/transfer_service/get_transfer_job_with_retries.py diff --git a/storage/transfer_service/get_transfer_job_with_retries.py b/storage/transfer_service/get_transfer_job_with_retries.py new file mode 100644 index 00000000000..3bd8f2ad501 --- /dev/null +++ b/storage/transfer_service/get_transfer_job_with_retries.py @@ -0,0 +1,41 @@ +#!/usr/bin/env + +# Copyright 2021 Google LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START storagetransfer_get_latest_transfer_operation] + +"""Command-line sample that checks the latest operation of a transfer. +This sample is used on this page: + https://cloud.google.com/storage/transfer/create-transfer +For more information, see README.md. +""" + +import googleapiclient.discovery + + +def get_transfer_job_with_retries(project_id, job_name, retries): + """Check the latest transfer operation associated with a transfer job.""" + storagetransfer = googleapiclient.discovery.build("storagetransfer", "v1") + + transferJob = ( + storagetransfer.transferJobs() + .get(projectId=project_id, jobName=job_name) + .execute(num_retries=retries) + ) + print( + "Fetched transfer job: " + + transferJob.get("name") + + " using {} retries".format(retries) + ) diff --git a/storage/transfer_service/sts_snippets_test.py b/storage/transfer_service/sts_snippets_test.py index 4338e8ded26..8b591c70b3d 100644 --- a/storage/transfer_service/sts_snippets_test.py +++ b/storage/transfer_service/sts_snippets_test.py @@ -17,6 +17,7 @@ import googleapiclient.discovery import check_latest_transfer_operation +import get_transfer_job_with_retries def test_latest_transfer_operation(capsys): @@ -50,3 +51,39 @@ def test_latest_transfer_operation(capsys): # The latest operation field can take a while to populate, so to avoid a # flaky test we just check that the job exists and the field was checked assert job_name in out + + +def test_get_transfer_Job_with_retries(capsys): + project_id = os.environ["GOOGLE_CLOUD_PROJECT"] + + transfer_job = { + "description": "Sample job", + "status": "ENABLED", + "projectId": project_id, + "schedule": { + "scheduleStartDate": {"day": "01", "month": "01", "year": "2000"}, + "startTimeOfDay": {"hours": "00", "minutes": "00", "seconds": "00"}, + }, + "transferSpec": { + "gcsDataSource": {"bucketName": project_id + "-storagetransfer-source"}, + "gcsDataSink": {"bucketName": project_id + "-storagetransfer-sink"}, + "objectConditions": { + "minTimeElapsedSinceLastModification": "2592000s" # 30 days + }, + "transferOptions": {"deleteObjectsFromSourceAfterTransfer": "true"}, + }, + } + storagetransfer = googleapiclient.discovery.build("storagetransfer", "v1") + result = storagetransfer.transferJobs().create(body=transfer_job).execute() + + job_name = result.get("name") + + retries = 3 + + get_transfer_job_with_retries.get_transfer_job_with_retries( + project_id, job_name, retries + ) + out, _ = capsys.readouterr() + # This sample isn't really meant to do anything, just check that it ran without any issues + # when we populated num_retries + assert str(retries) in out From bca3cb62c7d374cdeb308e5ba1acb9611bb3993d Mon Sep 17 00:00:00 2001 From: Jesse Lovelace Date: Wed, 28 Apr 2021 10:26:16 -0700 Subject: [PATCH 2/3] add region tag --- storage/transfer_service/get_transfer_job_with_retries.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/storage/transfer_service/get_transfer_job_with_retries.py b/storage/transfer_service/get_transfer_job_with_retries.py index 3bd8f2ad501..1d162d0942c 100644 --- a/storage/transfer_service/get_transfer_job_with_retries.py +++ b/storage/transfer_service/get_transfer_job_with_retries.py @@ -14,12 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -# [START storagetransfer_get_latest_transfer_operation] +# [START storagetransfer_create_retry_handler] -"""Command-line sample that checks the latest operation of a transfer. -This sample is used on this page: - https://cloud.google.com/storage/transfer/create-transfer -For more information, see README.md. +"""Command-line sample that gets a transfer job using retries """ import googleapiclient.discovery @@ -39,3 +36,4 @@ def get_transfer_job_with_retries(project_id, job_name, retries): + transferJob.get("name") + " using {} retries".format(retries) ) +# [END storagetransfer_create_retry_handler] From 24050bc4d74eb08f13af645266a60a786770408c Mon Sep 17 00:00:00 2001 From: "Leah E. Cole" <6719667+leahecole@users.noreply.github.com> Date: Fri, 21 May 2021 11:12:48 -0700 Subject: [PATCH 3/3] Update storage/transfer_service/sts_snippets_test.py --- storage/transfer_service/sts_snippets_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/transfer_service/sts_snippets_test.py b/storage/transfer_service/sts_snippets_test.py index 8b591c70b3d..e7ff2e90400 100644 --- a/storage/transfer_service/sts_snippets_test.py +++ b/storage/transfer_service/sts_snippets_test.py @@ -86,4 +86,4 @@ def test_get_transfer_Job_with_retries(capsys): out, _ = capsys.readouterr() # This sample isn't really meant to do anything, just check that it ran without any issues # when we populated num_retries - assert str(retries) in out + assert f"using {str(retries)} retries" in out