From cd695e37f1f34a70b76cda578099b753a77c5983 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Thu, 14 Apr 2022 01:54:21 +0200 Subject: [PATCH 01/28] chore(deps): update dependency google-cloud-storage to v2.3.0 (#766) --- samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 9074c1573..d87433b82 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,4 +1,4 @@ google-cloud-pubsub==2.12.0 -google-cloud-storage==2.2.1 +google-cloud-storage==2.3.0 pandas===1.3.5; python_version == '3.7' pandas==1.4.2; python_version >= '3.8' From 50ef911b1e8c9d8cd9852d729ecd84103bddc5c4 Mon Sep 17 00:00:00 2001 From: Tianzi Cai Date: Thu, 14 Apr 2022 13:48:07 -0700 Subject: [PATCH 02/28] samples(docs): remove beta tag in gcloud command (#767) --- samples/snippets/notification_polling.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/notification_polling.py b/samples/snippets/notification_polling.py index 3182db6da..34fd8cc3e 100644 --- a/samples/snippets/notification_polling.py +++ b/samples/snippets/notification_polling.py @@ -38,7 +38,7 @@ $ gsutil notification create -f json -t testtopic gs://testbucket 5. Create a subscription for your new topic: - $ gcloud beta pubsub subscriptions create testsubscription --topic=testtopic + $ gcloud pubsub subscriptions create testsubscription --topic=testtopic 6. Run this program: $ python notification_polling.py my-project-id testsubscription From b2e5150f191c04acb47ad98cef88512451aff81d Mon Sep 17 00:00:00 2001 From: Andrew Gorcester Date: Fri, 15 Apr 2022 10:42:12 -0700 Subject: [PATCH 03/28] feat: add AbortIncompleteMultipartUpload lifecycle rule (#765) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #753 🦕 --- google/cloud/storage/bucket.py | 58 +++++++++++++++++++++++++++--- tests/system/_helpers.py | 2 +- tests/system/test_bucket.py | 17 +++++++++ tests/unit/test_bucket.py | 65 +++++++++++++++++++++++++++++++++- 4 files changed, 136 insertions(+), 6 deletions(-) diff --git a/google/cloud/storage/bucket.py b/google/cloud/storage/bucket.py index 85c9302f7..be99ad141 100644 --- a/google/cloud/storage/bucket.py +++ b/google/cloud/storage/bucket.py @@ -323,7 +323,7 @@ class LifecycleRuleDelete(dict): def __init__(self, **kw): conditions = LifecycleRuleConditions(**kw) rule = {"action": {"type": "Delete"}, "condition": dict(conditions)} - super(LifecycleRuleDelete, self).__init__(rule) + super().__init__(rule) @classmethod def from_api_repr(cls, resource): @@ -356,7 +356,7 @@ def __init__(self, storage_class, **kw): "action": {"type": "SetStorageClass", "storageClass": storage_class}, "condition": dict(conditions), } - super(LifecycleRuleSetStorageClass, self).__init__(rule) + super().__init__(rule) @classmethod def from_api_repr(cls, resource): @@ -365,7 +365,7 @@ def from_api_repr(cls, resource): :type resource: dict :param resource: mapping as returned from API call. - :rtype: :class:`LifecycleRuleDelete` + :rtype: :class:`LifecycleRuleSetStorageClass` :returns: Instance created from resource. """ action = resource["action"] @@ -374,6 +374,38 @@ def from_api_repr(cls, resource): return instance +class LifecycleRuleAbortIncompleteMultipartUpload(dict): + """Map a rule aborting incomplete multipart uploads of matching items. + + The "age" lifecycle condition is the only supported condition for this rule. + + :type kw: dict + :params kw: arguments passed to :class:`LifecycleRuleConditions`. + """ + + def __init__(self, **kw): + conditions = LifecycleRuleConditions(**kw) + rule = { + "action": {"type": "AbortIncompleteMultipartUpload"}, + "condition": dict(conditions), + } + super().__init__(rule) + + @classmethod + def from_api_repr(cls, resource): + """Factory: construct instance from resource. + + :type resource: dict + :param resource: mapping as returned from API call. + + :rtype: :class:`LifecycleRuleAbortIncompleteMultipartUpload` + :returns: Instance created from resource. + """ + instance = cls(_factory=True) + instance.update(resource) + return instance + + _default = object() @@ -2240,6 +2272,8 @@ def lifecycle_rules(self): yield LifecycleRuleDelete.from_api_repr(rule) elif action_type == "SetStorageClass": yield LifecycleRuleSetStorageClass.from_api_repr(rule) + elif action_type == "AbortIncompleteMultipartUpload": + yield LifecycleRuleAbortIncompleteMultipartUpload.from_api_repr(rule) else: warnings.warn( "Unknown lifecycle rule type received: {}. Please upgrade to the latest version of google-cloud-storage.".format( @@ -2289,7 +2323,7 @@ def add_lifecycle_delete_rule(self, **kw): self.lifecycle_rules = rules def add_lifecycle_set_storage_class_rule(self, storage_class, **kw): - """Add a "delete" rule to lifestyle rules configured for this bucket. + """Add a "set storage class" rule to lifestyle rules. See https://cloud.google.com/storage/docs/lifecycle and https://cloud.google.com/storage/docs/json_api/v1/buckets @@ -2309,6 +2343,22 @@ def add_lifecycle_set_storage_class_rule(self, storage_class, **kw): rules.append(LifecycleRuleSetStorageClass(storage_class, **kw)) self.lifecycle_rules = rules + def add_lifecycle_abort_incomplete_multipart_upload_rule(self, **kw): + """Add a "abort incomplete multipart upload" rule to lifestyle rules. + + Note that the "age" lifecycle condition is the only supported condition + for this rule. + + See https://cloud.google.com/storage/docs/lifecycle and + https://cloud.google.com/storage/docs/json_api/v1/buckets + + :type kw: dict + :params kw: arguments passed to :class:`LifecycleRuleConditions`. + """ + rules = list(self.lifecycle_rules) + rules.append(LifecycleRuleAbortIncompleteMultipartUpload(**kw)) + self.lifecycle_rules = rules + _location = _scalar_property("location") @property diff --git a/tests/system/_helpers.py b/tests/system/_helpers.py index c172129d6..70c1f2a5d 100644 --- a/tests/system/_helpers.py +++ b/tests/system/_helpers.py @@ -23,7 +23,7 @@ retry_429 = RetryErrors(exceptions.TooManyRequests) retry_429_harder = RetryErrors(exceptions.TooManyRequests, max_tries=10) retry_429_503 = RetryErrors( - [exceptions.TooManyRequests, exceptions.ServiceUnavailable], max_tries=10 + (exceptions.TooManyRequests, exceptions.ServiceUnavailable), max_tries=10 ) retry_failures = RetryErrors(AssertionError) diff --git a/tests/system/test_bucket.py b/tests/system/test_bucket.py index 4826ce8a6..de1a04aa9 100644 --- a/tests/system/test_bucket.py +++ b/tests/system/test_bucket.py @@ -42,6 +42,7 @@ def test_bucket_lifecycle_rules(storage_client, buckets_to_delete): from google.cloud.storage import constants from google.cloud.storage.bucket import LifecycleRuleDelete from google.cloud.storage.bucket import LifecycleRuleSetStorageClass + from google.cloud.storage.bucket import LifecycleRuleAbortIncompleteMultipartUpload bucket_name = _helpers.unique_name("w-lifcycle-rules") custom_time_before = datetime.date(2018, 8, 1) @@ -64,6 +65,9 @@ def test_bucket_lifecycle_rules(storage_client, buckets_to_delete): is_live=False, matches_storage_class=[constants.NEARLINE_STORAGE_CLASS], ) + bucket.add_lifecycle_abort_incomplete_multipart_upload_rule( + age=42, + ) expected_rules = [ LifecycleRuleDelete( @@ -79,6 +83,9 @@ def test_bucket_lifecycle_rules(storage_client, buckets_to_delete): is_live=False, matches_storage_class=[constants.NEARLINE_STORAGE_CLASS], ), + LifecycleRuleAbortIncompleteMultipartUpload( + age=42, + ), ] _helpers.retry_429_503(bucket.create)(location="us") @@ -87,6 +94,16 @@ def test_bucket_lifecycle_rules(storage_client, buckets_to_delete): assert bucket.name == bucket_name assert list(bucket.lifecycle_rules) == expected_rules + # Test modifying lifecycle rules + expected_rules[0] = LifecycleRuleDelete(age=30) + rules = list(bucket.lifecycle_rules) + rules[0]["condition"] = {"age": 30} + bucket.lifecycle_rules = rules + bucket.patch() + + assert list(bucket.lifecycle_rules) == expected_rules + + # Test clearing lifecycle rules bucket.clear_lifecyle_rules() bucket.patch() diff --git a/tests/unit/test_bucket.py b/tests/unit/test_bucket.py index c5f1df5d2..eb402de9e 100644 --- a/tests/unit/test_bucket.py +++ b/tests/unit/test_bucket.py @@ -337,6 +337,43 @@ def test_from_api_repr(self): self.assertEqual(dict(rule), resource) +class Test_LifecycleRuleAbortIncompleteMultipartUpload(unittest.TestCase): + @staticmethod + def _get_target_class(): + from google.cloud.storage.bucket import ( + LifecycleRuleAbortIncompleteMultipartUpload, + ) + + return LifecycleRuleAbortIncompleteMultipartUpload + + def _make_one(self, **kw): + return self._get_target_class()(**kw) + + def test_ctor_wo_conditions(self): + with self.assertRaises(ValueError): + self._make_one() + + def test_ctor_w_condition(self): + rule = self._make_one(age=10) + expected = { + "action": {"type": "AbortIncompleteMultipartUpload"}, + "condition": {"age": 10}, + } + self.assertEqual(dict(rule), expected) + + def test_from_api_repr(self): + klass = self._get_target_class() + conditions = { + "age": 10, + } + resource = { + "action": {"type": "AbortIncompleteMultipartUpload"}, + "condition": conditions, + } + rule = klass.from_api_repr(resource) + self.assertEqual(dict(rule), resource) + + class Test_IAMConfiguration(unittest.TestCase): @staticmethod def _get_target_class(): @@ -2242,6 +2279,7 @@ def test_lifecycle_rules_getter(self): from google.cloud.storage.bucket import ( LifecycleRuleDelete, LifecycleRuleSetStorageClass, + LifecycleRuleAbortIncompleteMultipartUpload, ) NAME = "name" @@ -2250,7 +2288,11 @@ def test_lifecycle_rules_getter(self): "action": {"type": "SetStorageClass", "storageClass": "NEARLINE"}, "condition": {"isLive": False}, } - rules = [DELETE_RULE, SSC_RULE] + MULTIPART_RULE = { + "action": {"type": "AbortIncompleteMultipartUpload"}, + "condition": {"age": 42}, + } + rules = [DELETE_RULE, SSC_RULE, MULTIPART_RULE] properties = {"lifecycle": {"rule": rules}} bucket = self._make_one(name=NAME, properties=properties) @@ -2264,6 +2306,12 @@ def test_lifecycle_rules_getter(self): self.assertIsInstance(ssc_rule, LifecycleRuleSetStorageClass) self.assertEqual(dict(ssc_rule), SSC_RULE) + multipart_rule = found[2] + self.assertIsInstance( + multipart_rule, LifecycleRuleAbortIncompleteMultipartUpload + ) + self.assertEqual(dict(multipart_rule), MULTIPART_RULE) + def test_lifecycle_rules_setter_w_dicts(self): NAME = "name" DELETE_RULE = {"action": {"type": "Delete"}, "condition": {"age": 42}} @@ -2348,6 +2396,21 @@ def test_add_lifecycle_set_storage_class_rule(self): self.assertEqual([dict(rule) for rule in bucket.lifecycle_rules], rules) self.assertTrue("lifecycle" in bucket._changes) + def test_add_lifecycle_abort_incomplete_multipart_upload_rule(self): + NAME = "name" + AIMPU_RULE = { + "action": {"type": "AbortIncompleteMultipartUpload"}, + "condition": {"age": 42}, + } + rules = [AIMPU_RULE] + bucket = self._make_one(name=NAME) + self.assertEqual(list(bucket.lifecycle_rules), []) + + bucket.add_lifecycle_abort_incomplete_multipart_upload_rule(age=42) + + self.assertEqual([dict(rule) for rule in bucket.lifecycle_rules], rules) + self.assertTrue("lifecycle" in bucket._changes) + def test_cors_getter(self): NAME = "name" CORS_ENTRY = { From 01ddae8f13fb13cfa3768ae931c16207fa6a8406 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Wed, 20 Apr 2022 20:35:30 -0400 Subject: [PATCH 04/28] chore(python): add nox session to sort python imports (#774) Source-Link: https://github.com/googleapis/synthtool/commit/1b71c10e20de7ed3f97f692f99a0e3399b67049f Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:00c9d764fd1cd56265f12a5ef4b99a0c9e87cf261018099141e2ca5158890416 Co-authored-by: Owl Bot --- .github/.OwlBot.lock.yaml | 4 ++-- samples/snippets/noxfile.py | 21 +++++++++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index bc893c979..7c454abf7 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:8a5d3f6a2e43ed8293f34e06a2f56931d1e88a2694c3bb11b15df4eb256ad163 -# created: 2022-04-06T10:30:21.687684602Z + digest: sha256:00c9d764fd1cd56265f12a5ef4b99a0c9e87cf261018099141e2ca5158890416 +# created: 2022-04-20T23:42:53.970438194Z diff --git a/samples/snippets/noxfile.py b/samples/snippets/noxfile.py index 949e0fde9..38bb0a572 100644 --- a/samples/snippets/noxfile.py +++ b/samples/snippets/noxfile.py @@ -30,6 +30,7 @@ # WARNING - WARNING - WARNING - WARNING - WARNING BLACK_VERSION = "black==22.3.0" +ISORT_VERSION = "isort==5.10.1" # Copy `noxfile_config.py` to your directory and modify it instead. @@ -168,12 +169,32 @@ def lint(session: nox.sessions.Session) -> None: @nox.session def blacken(session: nox.sessions.Session) -> None: + """Run black. Format code to uniform standard.""" session.install(BLACK_VERSION) python_files = [path for path in os.listdir(".") if path.endswith(".py")] session.run("black", *python_files) +# +# format = isort + black +# + +@nox.session +def format(session: nox.sessions.Session) -> None: + """ + Run isort to sort imports. Then run black + to format code to uniform standard. + """ + session.install(BLACK_VERSION, ISORT_VERSION) + python_files = [path for path in os.listdir(".") if path.endswith(".py")] + + # Use the --fss option to sort imports using strict alphabetical order. + # See https://pycqa.github.io/isort/docs/configuration/options.html#force-sort-within-sections + session.run("isort", "--fss", *python_files) + session.run("black", *python_files) + + # # Sample Tests # From fbff4ef8c9d977a5123d6bad1d1cd0ec19d84a6b Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Thu, 21 Apr 2022 16:26:23 +0000 Subject: [PATCH 05/28] chore(python): use ubuntu 22.04 in docs image (#776) Source-Link: https://github.com/googleapis/synthtool/commit/f15cc72fb401b4861cedebb10af74afe428fb1f8 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:bc5eed3804aec2f05fad42aacf973821d9500c174015341f721a984a0825b6fd --- .github/.OwlBot.lock.yaml | 4 ++-- .kokoro/docker/docs/Dockerfile | 20 ++++++++++++++++++-- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 7c454abf7..64f82d6bf 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:00c9d764fd1cd56265f12a5ef4b99a0c9e87cf261018099141e2ca5158890416 -# created: 2022-04-20T23:42:53.970438194Z + digest: sha256:bc5eed3804aec2f05fad42aacf973821d9500c174015341f721a984a0825b6fd +# created: 2022-04-21T15:43:16.246106921Z diff --git a/.kokoro/docker/docs/Dockerfile b/.kokoro/docker/docs/Dockerfile index 4e1b1fb8b..238b87b9d 100644 --- a/.kokoro/docker/docs/Dockerfile +++ b/.kokoro/docker/docs/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ubuntu:20.04 +from ubuntu:22.04 ENV DEBIAN_FRONTEND noninteractive @@ -60,8 +60,24 @@ RUN apt-get update \ && rm -rf /var/lib/apt/lists/* \ && rm -f /var/cache/apt/archives/*.deb +###################### Install python 3.8.11 + +# Download python 3.8.11 +RUN wget https://www.python.org/ftp/python/3.8.11/Python-3.8.11.tgz + +# Extract files +RUN tar -xvf Python-3.8.11.tgz + +# Install python 3.8.11 +RUN ./Python-3.8.11/configure --enable-optimizations +RUN make altinstall + +###################### Install pip RUN wget -O /tmp/get-pip.py 'https://bootstrap.pypa.io/get-pip.py' \ - && python3.8 /tmp/get-pip.py \ + && python3 /tmp/get-pip.py \ && rm /tmp/get-pip.py +# Test pip +RUN python3 -m pip + CMD ["python3.8"] From 8de5b962829d445635fe513b50faffb946858b9c Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Mon, 25 Apr 2022 17:12:06 +0200 Subject: [PATCH 06/28] chore(deps): update dependency pytest to v7.1.2 (#777) --- samples/snippets/requirements-test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements-test.txt b/samples/snippets/requirements-test.txt index 6fa4b2753..cf4bca942 100644 --- a/samples/snippets/requirements-test.txt +++ b/samples/snippets/requirements-test.txt @@ -1,3 +1,3 @@ -pytest==7.1.1 +pytest==7.1.2 mock==4.0.3 backoff==1.11.1 \ No newline at end of file From 7ddfcba0d3008eb7ed1fc8cf8755386dd5afd396 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Tue, 26 Apr 2022 20:31:15 +0200 Subject: [PATCH 07/28] chore(deps): update dependency backoff to v2 (#778) --- samples/snippets/requirements-test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements-test.txt b/samples/snippets/requirements-test.txt index cf4bca942..c54832809 100644 --- a/samples/snippets/requirements-test.txt +++ b/samples/snippets/requirements-test.txt @@ -1,3 +1,3 @@ pytest==7.1.2 mock==4.0.3 -backoff==1.11.1 \ No newline at end of file +backoff==2.0.0 \ No newline at end of file From b452477ccc7818d2199bddb76151ce12b128721c Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Wed, 27 Apr 2022 18:10:35 +0200 Subject: [PATCH 08/28] chore(deps): update dependency backoff to v2.0.1 (#779) --- samples/snippets/requirements-test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements-test.txt b/samples/snippets/requirements-test.txt index c54832809..88beb7ba2 100644 --- a/samples/snippets/requirements-test.txt +++ b/samples/snippets/requirements-test.txt @@ -1,3 +1,3 @@ pytest==7.1.2 mock==4.0.3 -backoff==2.0.0 \ No newline at end of file +backoff==2.0.1 \ No newline at end of file From 36fce3262fb1c646ff9acd1f1b9eade48c13db58 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Thu, 5 May 2022 12:39:28 -0400 Subject: [PATCH 09/28] chore: [autoapprove] update readme_gen.py to include autoescape True (#781) Source-Link: https://github.com/googleapis/synthtool/commit/6b4d5a6407d740beb4158b302194a62a4108a8a6 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:f792ee1320e03eda2d13a5281a2989f7ed8a9e50b73ef6da97fac7e1e850b149 Co-authored-by: Owl Bot --- .github/.OwlBot.lock.yaml | 4 ++-- scripts/readme-gen/readme_gen.py | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 64f82d6bf..b631901e9 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:bc5eed3804aec2f05fad42aacf973821d9500c174015341f721a984a0825b6fd -# created: 2022-04-21T15:43:16.246106921Z + digest: sha256:f792ee1320e03eda2d13a5281a2989f7ed8a9e50b73ef6da97fac7e1e850b149 +# created: 2022-05-05T15:17:27.599381182Z diff --git a/scripts/readme-gen/readme_gen.py b/scripts/readme-gen/readme_gen.py index d309d6e97..91b59676b 100644 --- a/scripts/readme-gen/readme_gen.py +++ b/scripts/readme-gen/readme_gen.py @@ -28,7 +28,10 @@ jinja_env = jinja2.Environment( trim_blocks=True, loader=jinja2.FileSystemLoader( - os.path.abspath(os.path.join(os.path.dirname(__file__), 'templates')))) + os.path.abspath(os.path.join(os.path.dirname(__file__), "templates")) + ), + autoescape=True, +) README_TMPL = jinja_env.get_template('README.tmpl.rst') From c9bf5bfd9b8fc32b3c3565872df64110e006b77b Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Thu, 5 May 2022 23:04:20 +0000 Subject: [PATCH 10/28] chore(python): auto approve template changes (#782) Source-Link: https://github.com/googleapis/synthtool/commit/453a5d9c9a55d1969240a37d36cec626d20a9024 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:81ed5ecdfc7cac5b699ba4537376f3563f6f04122c4ec9e735d3b3dc1d43dd32 --- .github/.OwlBot.lock.yaml | 4 ++-- .github/auto-approve.yml | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 .github/auto-approve.yml diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index b631901e9..757c9dca7 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:f792ee1320e03eda2d13a5281a2989f7ed8a9e50b73ef6da97fac7e1e850b149 -# created: 2022-05-05T15:17:27.599381182Z + digest: sha256:81ed5ecdfc7cac5b699ba4537376f3563f6f04122c4ec9e735d3b3dc1d43dd32 +# created: 2022-05-05T22:08:23.383410683Z diff --git a/.github/auto-approve.yml b/.github/auto-approve.yml new file mode 100644 index 000000000..311ebbb85 --- /dev/null +++ b/.github/auto-approve.yml @@ -0,0 +1,3 @@ +# https://github.com/googleapis/repo-automation-bots/tree/main/packages/auto-approve +processes: + - "OwlBotTemplateChanges" From 841ab04f1b875317ec8bdba19675f4885602d4e8 Mon Sep 17 00:00:00 2001 From: cojenco Date: Tue, 10 May 2022 12:32:19 -0700 Subject: [PATCH 11/28] tests: move range read test (#787) Move range read test to system testing. Skip conf test as the testbench is now aligned with XML and returns complete object checksum. Range reads headers are returned differently in JSON vs XML. The JSON response omits the x-goog-hash header whereas the XML response returns checksum that covers the complete object content for range reads. --- tests/conformance/test_conformance.py | 11 ----------- tests/system/test_blob.py | 5 +++++ 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/tests/conformance/test_conformance.py b/tests/conformance/test_conformance.py index fa842630e..f84131f2f 100644 --- a/tests/conformance/test_conformance.py +++ b/tests/conformance/test_conformance.py @@ -85,16 +85,6 @@ def blob_download_as_bytes(client, _preconditions, **resources): assert stored_contents == data.encode("utf-8") -def blob_download_as_bytes_w_range(client, _preconditions, **resources): - bucket = resources.get("bucket") - file, data = resources.get("file_data") - blob = client.bucket(bucket.name).blob(file.name) - start_byte = 0 - end_byte = 1000000 - stored_contents = blob.download_as_bytes(start=start_byte, end=end_byte - 1) - assert stored_contents == data.encode("utf-8")[start_byte:end_byte] - - def blob_download_as_text(client, _preconditions, **resources): bucket = resources.get("bucket") file, data = resources.get("file_data") @@ -767,7 +757,6 @@ def object_acl_clear(client, _preconditions, **resources): blob_download_to_filename, blob_download_to_filename_chunked, blob_download_as_bytes, - blob_download_as_bytes_w_range, blob_download_as_text, blobreader_read, ], diff --git a/tests/system/test_blob.py b/tests/system/test_blob.py index acbc5745f..773dbdf81 100644 --- a/tests/system/test_blob.py +++ b/tests/system/test_blob.py @@ -614,6 +614,11 @@ def test_blob_download_as_text( assert stored_contents == payload assert blob.etag == etag + # Test download with byte range + end_byte = 5 + stored_contents = blob.download_as_text(start=0, end=end_byte - 1) + assert stored_contents == payload[0:end_byte] + def test_blob_upload_w_gzip_encoded_download_raw( shared_bucket, From 3c90d611cbc3b9944bbd46b0dc9f0d8fcb38d83d Mon Sep 17 00:00:00 2001 From: cojenco Date: Tue, 10 May 2022 14:53:01 -0700 Subject: [PATCH 12/28] chore: add internal benchmarking script (#760) * chore: add internal benchmarking script and readme * archive benchwrapper to subdirectory * blacken lint * fix typo * update script with preconditions and upload from disk * download to file * update multiprocessing and readme * clean up * update benchmarking script * update checksumming options and default num processes * replace tempfile package usage --- tests/perf/README.md | 48 ++- tests/perf/benchmarking.py | 274 ++++++++++++++++++ tests/perf/benchwrapper/README.md | 21 ++ tests/perf/{ => benchwrapper}/benchwrapper.py | 0 tests/perf/{ => benchwrapper}/storage.proto | 0 tests/perf/{ => benchwrapper}/storage_pb2.py | 0 .../{ => benchwrapper}/storage_pb2_grpc.py | 0 7 files changed, 331 insertions(+), 12 deletions(-) create mode 100644 tests/perf/benchmarking.py create mode 100644 tests/perf/benchwrapper/README.md rename tests/perf/{ => benchwrapper}/benchwrapper.py (100%) rename tests/perf/{ => benchwrapper}/storage.proto (100%) rename tests/perf/{ => benchwrapper}/storage_pb2.py (100%) rename tests/perf/{ => benchwrapper}/storage_pb2_grpc.py (100%) diff --git a/tests/perf/README.md b/tests/perf/README.md index e77589f61..d530b12d9 100644 --- a/tests/perf/README.md +++ b/tests/perf/README.md @@ -1,21 +1,45 @@ -# storage benchwrapp +# python-storage benchmarking -main.py is a gRPC wrapper around the storage library for benchmarking purposes. +**This is not an officially supported Google product** -## Running +This benchmarking script is used by Storage client library maintainers to benchmark various workloads and collect metrics in order to improve performance of the library. +Currently the benchmarking runs a Write-1-Read-3 workload and measures the usual two QoS performance attributes, latency and throughput. +## Run example: +This runs 10K iterations of Write-1-Read-3 on 5KiB to 16KiB files, and generates output to a default csv file `benchmarking.csv`: ```bash -$ export STORAGE_EMULATOR_HOST=http://localhost:8080 -$ pip install grpcio -$ cd storage +$ cd python-storage $ pip install -e . # install google.cloud.storage locally $ cd tests/perf -$ python3 benchwrapper.py --port 8081 +$ python3 benchmarking.py --num_samples 10000 --max_size 16384 ``` -## Re-generating protos +## CLI parameters -```bash -$ pip install grpcio-tools -$ python -m grpc_tools.protoc -I. --python_out=. --grpc_python_out=. *.proto -``` +| Parameter | Description | Possible values | Default | +| --------- | ----------- | --------------- |:-------:| +| --min_size | minimum object size in bytes | any positive integer | `5120` (5 KiB) | +| --max_size | maximum object size in bytes | any positive integer | `2147483648` (2 GiB) | +| --num_samples | number of W1R3 iterations | any positive integer | `1000` | +| --r | bucket region for benchmarks | any GCS region | `US` | +| --p | number of processes (multiprocessing enabled) | any positive integer | 16 (recommend not to exceed 16) | +| --o | file to output results to | any file path | `benchmarking.csv` | + + +## Workload definition and CSV headers + +For each invocation of the benchmark, write a new object of random size between `min_size` and `max_size` . After the successful write, download the object in full three times. For each of the 4 operations record the following fields: + +| Field | Description | +| ----- | ----------- | +| Op | the name of the operations (WRITE, READ[{0,1,2}]) | +| ObjectSize | the number of bytes of the object | +| LibBufferSize | configured to use the [library default of 100 MiB](https://github.com/googleapis/python-storage/blob/main/google/cloud/storage/blob.py#L135) | +| Crc32cEnabled | bool: whether crc32c was computed for the operation | +| MD5Enabled | bool: whether MD5 was computed for the operation | +| ApiName | default to JSON| +| ElapsedTimeUs | the elapsed time in microseconds the operation took | +| Status | completion state of the operation [OK, FAIL] | +| RunID | timestamp from the benchmarking run | +| AppBufferSize | N/A | +| CpuTimeUs | N/A | \ No newline at end of file diff --git a/tests/perf/benchmarking.py b/tests/perf/benchmarking.py new file mode 100644 index 000000000..2389b00e6 --- /dev/null +++ b/tests/perf/benchmarking.py @@ -0,0 +1,274 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Performance benchmarking script. This is not an officially supported Google product.""" + +import argparse +import csv +import logging +import multiprocessing +import os +import random +import time +import uuid + +from functools import partial, update_wrapper + +from google.cloud import storage + + +##### DEFAULTS & CONSTANTS ##### +HEADER = [ + "Op", + "ObjectSize", + "AppBufferSize", + "LibBufferSize", + "Crc32cEnabled", + "MD5Enabled", + "ApiName", + "ElapsedTimeUs", + "CpuTimeUs", + "Status", + "RunID", +] +CHECKSUM = ["md5", "crc32c", None] +TIMESTAMP = time.strftime("%Y%m%d-%H%M%S") +DEFAULT_API = "JSON" +DEFAULT_BUCKET_LOCATION = "US" +DEFAULT_MIN_SIZE = 5120 # 5 KiB +DEFAULT_MAX_SIZE = 2147483648 # 2 GiB +DEFAULT_NUM_SAMPLES = 1000 +DEFAULT_NUM_PROCESSES = 16 +DEFAULT_LIB_BUFFER_SIZE = 104857600 # https://github.com/googleapis/python-storage/blob/main/google/cloud/storage/blob.py#L135 +NOT_SUPPORTED = -1 + + +def log_performance(func): + """Log latency and throughput output per operation call.""" + # Holds benchmarking results for each operation + res = { + "ApiName": DEFAULT_API, + "RunID": TIMESTAMP, + "CpuTimeUs": NOT_SUPPORTED, + "AppBufferSize": NOT_SUPPORTED, + "LibBufferSize": DEFAULT_LIB_BUFFER_SIZE, + } + + try: + elapsed_time = func() + except Exception as e: + logging.exception( + f"Caught an exception while running operation {func.__name__}\n {e}" + ) + res["Status"] = ["FAIL"] + elapsed_time = NOT_SUPPORTED + else: + res["Status"] = ["OK"] + + checksum = func.keywords.get("checksum") + num = func.keywords.get("num", None) + res["ElapsedTimeUs"] = elapsed_time + res["ObjectSize"] = func.keywords.get("size") + res["Crc32cEnabled"] = checksum == "crc32c" + res["MD5Enabled"] = checksum == "md5" + res["Op"] = func.__name__ + if res["Op"] == "READ": + res["Op"] += f"[{num}]" + + return [ + res["Op"], + res["ObjectSize"], + res["AppBufferSize"], + res["LibBufferSize"], + res["Crc32cEnabled"], + res["MD5Enabled"], + res["ApiName"], + res["ElapsedTimeUs"], + res["CpuTimeUs"], + res["Status"], + res["RunID"], + ] + + +def WRITE(bucket, blob_name, checksum, size, **kwargs): + """Perform an upload and return latency.""" + blob = bucket.blob(blob_name) + file_path = f"{os.getcwd()}/{uuid.uuid4().hex}" + # Create random file locally on disk + with open(file_path, "wb") as file_obj: + file_obj.write(os.urandom(size)) + + start_time = time.monotonic_ns() + blob.upload_from_filename(file_path, checksum=checksum, if_generation_match=0) + end_time = time.monotonic_ns() + + elapsed_time = round( + (end_time - start_time) / 1000 + ) # convert nanoseconds to microseconds + + # Clean up local file + cleanup_file(file_path) + + return elapsed_time + + +def READ(bucket, blob_name, checksum, **kwargs): + """Perform a download and return latency.""" + blob = bucket.blob(blob_name) + if not blob.exists(): + raise Exception("Blob does not exist. Previous WRITE failed.") + + file_path = f"{os.getcwd()}/{blob_name}" + with open(file_path, "wb") as file_obj: + start_time = time.monotonic_ns() + blob.download_to_file(file_obj, checksum=checksum) + end_time = time.monotonic_ns() + + elapsed_time = round( + (end_time - start_time) / 1000 + ) # convert nanoseconds to microseconds + + # Clean up local file + cleanup_file(file_path) + + return elapsed_time + + +def cleanup_file(file_path): + """Clean up local file on disk.""" + try: + os.remove(file_path) + except Exception as e: + logging.exception(f"Caught an exception while deleting local file\n {e}") + + +def _wrapped_partial(func, *args, **kwargs): + """Helper method to create partial and propagate function name and doc from original function.""" + partial_func = partial(func, *args, **kwargs) + update_wrapper(partial_func, func) + return partial_func + + +def _generate_func_list(bucket_name, min_size, max_size): + """Generate Write-1-Read-3 workload.""" + # generate randmon size in bytes using a uniform distribution + size = random.randrange(min_size, max_size) + blob_name = f"{TIMESTAMP}-{uuid.uuid4().hex}" + + # generate random checksumming type: md5, crc32c or None + idx_checksum = random.choice([0, 1, 2]) + checksum = CHECKSUM[idx_checksum] + + func_list = [ + _wrapped_partial( + WRITE, + storage.Client().bucket(bucket_name), + blob_name, + size=size, + checksum=checksum, + ), + *[ + _wrapped_partial( + READ, + storage.Client().bucket(bucket_name), + blob_name, + size=size, + checksum=checksum, + num=i, + ) + for i in range(3) + ], + ] + return func_list + + +def benchmark_runner(args): + """Run benchmarking iterations.""" + results = [] + for func in _generate_func_list(args.b, args.min_size, args.max_size): + results.append(log_performance(func)) + + return results + + +def main(args): + # Create a storage bucket to run benchmarking + client = storage.Client() + if not client.bucket(args.b).exists(): + bucket = client.create_bucket(args.b, location=args.r) + + # Launch benchmark_runner using multiprocessing + p = multiprocessing.Pool(args.p) + pool_output = p.map(benchmark_runner, [args for _ in range(args.num_samples)]) + + # Output to CSV file + with open(args.o, "w") as file: + writer = csv.writer(file) + writer.writerow(HEADER) + for result in pool_output: + for row in result: + writer.writerow(row) + print(f"Succesfully ran benchmarking. Please find your output log at {args.o}") + + # Cleanup and delete bucket + try: + bucket.delete(force=True) + except Exception as e: + logging.exception(f"Caught an exception while deleting bucket\n {e}") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--min_size", + type=int, + default=DEFAULT_MIN_SIZE, + help="Minimum object size in bytes", + ) + parser.add_argument( + "--max_size", + type=int, + default=DEFAULT_MAX_SIZE, + help="Maximum object size in bytes", + ) + parser.add_argument( + "--num_samples", + type=int, + default=DEFAULT_NUM_SAMPLES, + help="Number of iterations", + ) + parser.add_argument( + "--p", + type=int, + default=DEFAULT_NUM_PROCESSES, + help="Number of processes- multiprocessing enabled", + ) + parser.add_argument( + "--r", type=str, default=DEFAULT_BUCKET_LOCATION, help="Bucket location" + ) + parser.add_argument( + "--o", + type=str, + default=f"benchmarking{TIMESTAMP}.csv", + help="File to output results to", + ) + parser.add_argument( + "--b", + type=str, + default=f"benchmarking{TIMESTAMP}", + help="Storage bucket name", + ) + args = parser.parse_args() + + main(args) diff --git a/tests/perf/benchwrapper/README.md b/tests/perf/benchwrapper/README.md new file mode 100644 index 000000000..e77589f61 --- /dev/null +++ b/tests/perf/benchwrapper/README.md @@ -0,0 +1,21 @@ +# storage benchwrapp + +main.py is a gRPC wrapper around the storage library for benchmarking purposes. + +## Running + +```bash +$ export STORAGE_EMULATOR_HOST=http://localhost:8080 +$ pip install grpcio +$ cd storage +$ pip install -e . # install google.cloud.storage locally +$ cd tests/perf +$ python3 benchwrapper.py --port 8081 +``` + +## Re-generating protos + +```bash +$ pip install grpcio-tools +$ python -m grpc_tools.protoc -I. --python_out=. --grpc_python_out=. *.proto +``` diff --git a/tests/perf/benchwrapper.py b/tests/perf/benchwrapper/benchwrapper.py similarity index 100% rename from tests/perf/benchwrapper.py rename to tests/perf/benchwrapper/benchwrapper.py diff --git a/tests/perf/storage.proto b/tests/perf/benchwrapper/storage.proto similarity index 100% rename from tests/perf/storage.proto rename to tests/perf/benchwrapper/storage.proto diff --git a/tests/perf/storage_pb2.py b/tests/perf/benchwrapper/storage_pb2.py similarity index 100% rename from tests/perf/storage_pb2.py rename to tests/perf/benchwrapper/storage_pb2.py diff --git a/tests/perf/storage_pb2_grpc.py b/tests/perf/benchwrapper/storage_pb2_grpc.py similarity index 100% rename from tests/perf/storage_pb2_grpc.py rename to tests/perf/benchwrapper/storage_pb2_grpc.py From 235171001287eb77003906f0fa4b83132ecada49 Mon Sep 17 00:00:00 2001 From: cojenco Date: Thu, 12 May 2022 09:31:09 -0700 Subject: [PATCH 13/28] chore: publish docs for missing modules (#784) --- docs/fileio.rst | 6 ++++++ docs/index.rst | 2 ++ docs/retry.rst | 6 ++++++ docs/retry_timeout.rst | 8 -------- google/cloud/storage/constants.py | 2 +- 5 files changed, 15 insertions(+), 9 deletions(-) create mode 100644 docs/fileio.rst create mode 100644 docs/retry.rst diff --git a/docs/fileio.rst b/docs/fileio.rst new file mode 100644 index 000000000..9ad214a25 --- /dev/null +++ b/docs/fileio.rst @@ -0,0 +1,6 @@ +FileIO +~~~~~~~ + +.. automodule:: google.cloud.storage.fileio + :members: + :show-inheritance: \ No newline at end of file diff --git a/docs/index.rst b/docs/index.rst index 777926af3..154c76d5e 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -18,9 +18,11 @@ API Reference buckets acl batch + fileio constants hmac_key notification + retry retry_timeout generation_metageneration diff --git a/docs/retry.rst b/docs/retry.rst new file mode 100644 index 000000000..bb5690539 --- /dev/null +++ b/docs/retry.rst @@ -0,0 +1,6 @@ +Retry +---------------- + +.. automodule:: google.cloud.storage.retry + :members: + :show-inheritance: \ No newline at end of file diff --git a/docs/retry_timeout.rst b/docs/retry_timeout.rst index 7c3ad3084..db072013b 100644 --- a/docs/retry_timeout.rst +++ b/docs/retry_timeout.rst @@ -142,11 +142,3 @@ explicit policy in your code. my_cond_policy = ConditionalRetryPolicy( my_retry_policy, conditional_predicate=is_etag_in_data) bucket = client.get_bucket(BUCKET_NAME, retry=my_cond_policy) - - -Retry Module API ----------------- - -.. automodule:: google.cloud.storage.retry - :members: - :show-inheritance: diff --git a/google/cloud/storage/constants.py b/google/cloud/storage/constants.py index 132f4e40a..b8ac87886 100644 --- a/google/cloud/storage/constants.py +++ b/google/cloud/storage/constants.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Constants used acros google.cloud.storage modules.""" +"""Constants used across google.cloud.storage modules.""" # Storage classes From 1b4413cc086dd8b0045469e3bbecce98e7d6051d Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Thu, 12 May 2022 21:48:18 +0200 Subject: [PATCH 14/28] chore(deps): update dependency google-cloud-pubsub to v2.12.1 (#788) Co-authored-by: Anthonios Partheniou --- samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index d87433b82..843c608cd 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,4 +1,4 @@ -google-cloud-pubsub==2.12.0 +google-cloud-pubsub==2.12.1 google-cloud-storage==2.3.0 pandas===1.3.5; python_version == '3.7' pandas==1.4.2; python_version >= '3.8' From 3664ddebe8746d0395acd86d5efa1ef973eeac3d Mon Sep 17 00:00:00 2001 From: Pal Szabo Date: Mon, 16 May 2022 17:39:39 +0200 Subject: [PATCH 15/28] cleanup: f-string formatting (#789) * cleanup: f-string formatting * cleanup: f-string formatting * remove unnecessary :d specifier Co-authored-by: Mariatta Wijaya * remove unnecessary :d specifier Co-authored-by: Mariatta Wijaya * cleanup: f-string formatting * cleanup: f-string formatting Co-authored-by: Mariatta Wijaya --- docs/snippets.py | 12 +- google/cloud/storage/_helpers.py | 8 +- google/cloud/storage/_http.py | 4 +- google/cloud/storage/_signing.py | 14 +- google/cloud/storage/acl.py | 12 +- google/cloud/storage/batch.py | 16 +-- google/cloud/storage/blob.py | 32 ++--- google/cloud/storage/bucket.py | 26 ++-- google/cloud/storage/client.py | 10 +- google/cloud/storage/fileio.py | 10 +- google/cloud/storage/hmac_key.py | 6 +- google/cloud/storage/notification.py | 8 +- samples/snippets/acl_test.py | 4 +- samples/snippets/bucket_lock_test.py | 10 +- samples/snippets/encryption_test.py | 4 +- samples/snippets/fileio_test.py | 4 +- samples/snippets/iam_test.py | 4 +- samples/snippets/notification_polling.py | 12 +- samples/snippets/notification_test.py | 2 +- samples/snippets/noxfile.py | 4 +- samples/snippets/quickstart.py | 2 +- samples/snippets/requester_pays_test.py | 6 +- samples/snippets/rpo_test.py | 4 +- samples/snippets/snippets_test.py | 20 +-- samples/snippets/storage_activate_hmac_key.py | 16 +-- ...rage_add_bucket_conditional_iam_binding.py | 10 +- .../snippets/storage_add_bucket_iam_member.py | 2 +- samples/snippets/storage_add_bucket_label.py | 2 +- samples/snippets/storage_add_bucket_owner.py | 4 +- .../storage_bucket_delete_default_kms_key.py | 2 +- .../storage_change_default_storage_class.py | 2 +- samples/snippets/storage_configure_retries.py | 2 +- .../snippets/storage_cors_configuration.py | 2 +- samples/snippets/storage_create_bucket.py | 2 +- samples/snippets/storage_create_hmac_key.py | 18 +-- .../snippets/storage_deactivate_hmac_key.py | 16 +-- samples/snippets/storage_delete_bucket.py | 2 +- samples/snippets/storage_delete_file.py | 2 +- ...storage_delete_file_archived_generation.py | 4 +- ...age_disable_bucket_lifecycle_management.py | 2 +- ...torage_disable_default_event_based_hold.py | 2 +- .../storage_disable_requester_pays.py | 2 +- ...age_disable_uniform_bucket_level_access.py | 2 +- .../snippets/storage_disable_versioning.py | 2 +- .../storage_download_encrypted_file.py | 4 +- .../snippets/storage_download_to_stream.py | 2 +- ...rage_enable_bucket_lifecycle_management.py | 4 +- ...storage_enable_default_event_based_hold.py | 2 +- .../snippets/storage_enable_requester_pays.py | 2 +- ...rage_enable_uniform_bucket_level_access.py | 2 +- samples/snippets/storage_enable_versioning.py | 2 +- .../storage_generate_encryption_key.py | 2 +- .../storage_generate_signed_post_policy_v4.py | 2 +- .../storage_generate_signed_url_v2.py | 2 +- .../storage_generate_signed_url_v4.py | 2 +- .../storage_get_default_event_based_hold.py | 6 +- samples/snippets/storage_get_hmac_key.py | 16 +-- samples/snippets/storage_get_metadata.py | 46 ++++--- .../storage_get_requester_pays_status.py | 4 +- .../snippets/storage_get_retention_policy.py | 6 +- .../snippets/storage_get_service_account.py | 4 +- ...storage_get_uniform_bucket_level_access.py | 8 +- .../storage_list_file_archived_generations.py | 2 +- samples/snippets/storage_list_hmac_keys.py | 4 +- .../snippets/storage_lock_retention_policy.py | 6 +- samples/snippets/storage_make_public.py | 4 +- .../snippets/storage_object_get_kms_key.py | 2 +- samples/snippets/storage_print_bucket_acl.py | 2 +- samples/snippets/storage_print_file_acl.py | 2 +- .../storage_release_event_based_hold.py | 2 +- .../storage_remove_bucket_default_owner.py | 4 +- .../storage_remove_bucket_iam_member.py | 2 +- .../snippets/storage_remove_bucket_label.py | 2 +- .../snippets/storage_remove_bucket_owner.py | 2 +- .../storage_remove_cors_configuration.py | 2 +- samples/snippets/storage_remove_file_owner.py | 4 +- .../storage_remove_retention_policy.py | 2 +- samples/snippets/storage_rename_file.py | 2 +- .../snippets/storage_rotate_encryption_key.py | 2 +- .../snippets/storage_set_bucket_public_iam.py | 2 +- .../snippets/storage_set_event_based_hold.py | 2 +- samples/snippets/storage_set_metadata.py | 2 +- .../snippets/storage_upload_encrypted_file.py | 4 +- samples/snippets/storage_upload_file.py | 4 +- .../snippets/storage_upload_from_memory.py | 4 +- .../snippets/storage_upload_from_stream.py | 4 +- .../storage_view_bucket_iam_members.py | 2 +- .../uniform_bucket_level_access_test.py | 6 +- tests/conformance/test_conformance.py | 24 ++-- tests/system/test_bucket.py | 2 +- tests/system/test_notification.py | 4 +- tests/unit/test__http.py | 8 +- tests/unit/test__signing.py | 18 +-- tests/unit/test_acl.py | 32 ++--- tests/unit/test_batch.py | 10 +- tests/unit/test_blob.py | 122 ++++++++---------- tests/unit/test_bucket.py | 72 +++++------ tests/unit/test_client.py | 46 +++---- tests/unit/test_hmac_key.py | 26 ++-- tests/unit/test_notification.py | 6 +- 100 files changed, 403 insertions(+), 489 deletions(-) diff --git a/docs/snippets.py b/docs/snippets.py index 89f92a20b..7ee3a62a0 100644 --- a/docs/snippets.py +++ b/docs/snippets.py @@ -260,9 +260,7 @@ def policy_document(client): # Generate an upload form using the form fields. policy_fields = "".join( - ''.format( - key=key, value=value - ) + f'' for key, value in policy.items() ) @@ -301,13 +299,15 @@ def main(): client = storage.Client() for example in _find_examples(): to_delete = [] - print("%-25s: %s" % _name_and_doc(example)) + name, doc = _name_and_doc(example) + print(f"{name:>25}: {doc}") + try: example(client, to_delete) except AssertionError as failure: - print(" FAIL: %s" % (failure,)) + print(f" FAIL: {failure}") except Exception as error: # pylint: disable=broad-except - print(" ERROR: %r" % (error,)) + print(f" ERROR: {error!r}") for item in to_delete: item.delete() diff --git a/google/cloud/storage/_helpers.py b/google/cloud/storage/_helpers.py index cc85525d8..282d9bcfb 100644 --- a/google/cloud/storage/_helpers.py +++ b/google/cloud/storage/_helpers.py @@ -519,14 +519,12 @@ def _raise_if_more_than_one_set(**kwargs): :raises: :class:`~ValueError` containing the fields that were set """ if sum(arg is not None for arg in kwargs.values()) > 1: - escaped_keys = ["'%s'" % name for name in kwargs.keys()] + escaped_keys = [f"'{name}'" for name in kwargs.keys()] keys_but_last = ", ".join(escaped_keys[:-1]) last_key = escaped_keys[-1] - msg = "Pass at most one of {keys_but_last} and {last_key}".format( - keys_but_last=keys_but_last, last_key=last_key - ) + msg = f"Pass at most one of {keys_but_last} and {last_key}" raise ValueError(msg) @@ -548,7 +546,7 @@ def _bucket_bound_hostname_url(host, scheme=None): if url_parts.scheme and url_parts.netloc: return host - return "{scheme}://{host}/".format(scheme=scheme, host=host) + return f"{scheme}://{host}/" def _api_core_retry_to_resumable_media_retry(retry, num_retries=None): diff --git a/google/cloud/storage/_http.py b/google/cloud/storage/_http.py index 9b29f6280..3a739bba6 100644 --- a/google/cloud/storage/_http.py +++ b/google/cloud/storage/_http.py @@ -48,9 +48,9 @@ def __init__(self, client, client_info=None, api_endpoint=None): # TODO: When metrics all use gccl, this should be removed #9552 if self._client_info.user_agent is None: # pragma: no branch self._client_info.user_agent = "" - agent_version = "gcloud-python/{}".format(__version__) + agent_version = f"gcloud-python/{__version__}" if agent_version not in self._client_info.user_agent: - self._client_info.user_agent += " {} ".format(agent_version) + self._client_info.user_agent += f" {agent_version} " API_VERSION = "v1" """The version of the API, used in building the API call's URL.""" diff --git a/google/cloud/storage/_signing.py b/google/cloud/storage/_signing.py index 837ef6211..036ea6385 100644 --- a/google/cloud/storage/_signing.py +++ b/google/cloud/storage/_signing.py @@ -157,9 +157,7 @@ def get_expiration_seconds_v4(expiration): seconds = int(expiration.total_seconds()) if seconds > SEVEN_DAYS: - raise ValueError( - "Max allowed expiration interval is seven days {}".format(SEVEN_DAYS) - ) + raise ValueError(f"Max allowed expiration interval is seven days {SEVEN_DAYS}") return seconds @@ -252,7 +250,7 @@ def canonicalize_v2(method, resource, query_parameters, headers): for key, value in query_parameters.items() ) encoded_qp = urllib.parse.urlencode(normalized_qp) - canonical_resource = "{}?{}".format(resource, encoded_qp) + canonical_resource = f"{resource}?{encoded_qp}" return _Canonical(method, canonical_resource, normalized_qp, headers) @@ -550,8 +548,8 @@ def generate_signed_url_v4( ensure_signed_credentials(credentials) client_email = credentials.signer_email - credential_scope = "{}/auto/storage/goog4_request".format(datestamp) - credential = "{}/{}".format(client_email, credential_scope) + credential_scope = f"{datestamp}/auto/storage/goog4_request" + credential = f"{client_email}/{credential_scope}" if headers is None: headers = {} @@ -689,7 +687,7 @@ def _sign_message(message, access_token, service_account_email): if response.status != http.client.OK: raise exceptions.TransportError( - "Error calling the IAM signBytes API: {}".format(response.data) + f"Error calling the IAM signBytes API: {response.data}" ) data = json.loads(response.data.decode("utf-8")) @@ -706,7 +704,7 @@ def _url_encode(query_params): :returns: URL encoded query params. """ params = [ - "{}={}".format(_quote_param(name), _quote_param(value)) + f"{_quote_param(name)}={_quote_param(value)}" for name, value in query_params.items() ] diff --git a/google/cloud/storage/acl.py b/google/cloud/storage/acl.py index ef2bca356..e876c152c 100644 --- a/google/cloud/storage/acl.py +++ b/google/cloud/storage/acl.py @@ -120,9 +120,7 @@ def __str__(self): return "{acl.type}-{acl.identifier}".format(acl=self) def __repr__(self): - return "".format( - acl=self, roles=", ".join(self.roles) - ) + return f"" def get_roles(self): """Get the list of roles permitted by this entity. @@ -242,7 +240,7 @@ def validate_predefined(cls, predefined): """ predefined = cls.PREDEFINED_XML_ACLS.get(predefined, predefined) if predefined and predefined not in cls.PREDEFINED_JSON_ACLS: - raise ValueError("Invalid predefined ACL: %s" % (predefined,)) + raise ValueError(f"Invalid predefined ACL: {predefined}") return predefined def reset(self): @@ -285,7 +283,7 @@ def entity_from_dict(self, entity_dict): entity = self.entity(entity_type=entity_type, identifier=identifier) if not isinstance(entity, _ACLEntity): - raise ValueError("Invalid dictionary: %s" % entity_dict) + raise ValueError(f"Invalid dictionary: {entity_dict}") entity.grant(role) return entity @@ -770,7 +768,7 @@ def client(self): @property def reload_path(self): """Compute the path for GET API requests for this ACL.""" - return "%s/%s" % (self.bucket.path, self._URL_PATH_ELEM) + return f"{self.bucket.path}/{self._URL_PATH_ELEM}" @property def save_path(self): @@ -809,7 +807,7 @@ def client(self): @property def reload_path(self): """Compute the path for GET API requests for this ACL.""" - return "%s/acl" % self.blob.path + return f"{self.blob.path}/acl" @property def save_path(self): diff --git a/google/cloud/storage/batch.py b/google/cloud/storage/batch.py index cbc93397f..599aa3a7f 100644 --- a/google/cloud/storage/batch.py +++ b/google/cloud/storage/batch.py @@ -57,10 +57,8 @@ def __init__(self, method, uri, headers, body): headers["Content-Length"] = len(body) if body is None: body = "" - lines = ["%s %s HTTP/1.1" % (method, uri)] - lines.extend( - ["%s: %s" % (key, value) for key, value in sorted(headers.items())] - ) + lines = [f"{method} {uri} HTTP/1.1"] + lines.extend([f"{key}: {value}" for key, value in sorted(headers.items())]) lines.append("") lines.append(body) payload = "\r\n".join(lines) @@ -86,7 +84,7 @@ def get(key, default=None): :raises: :class:`KeyError` always since the future is intended to fail as a dictionary. """ - raise KeyError("Cannot get(%r, default=%r) on a future" % (key, default)) + raise KeyError(f"Cannot get({key!r}, default={default!r}) on a future") def __getitem__(self, key): """Stand-in for dict[key]. @@ -97,7 +95,7 @@ def __getitem__(self, key): :raises: :class:`KeyError` always since the future is intended to fail as a dictionary. """ - raise KeyError("Cannot get item %r from a future" % (key,)) + raise KeyError(f"Cannot get item {key!r} from a future") def __setitem__(self, key, value): """Stand-in for dict[key] = value. @@ -111,7 +109,7 @@ def __setitem__(self, key, value): :raises: :class:`KeyError` always since the future is intended to fail as a dictionary. """ - raise KeyError("Cannot set %r -> %r on a future" % (key, value)) + raise KeyError(f"Cannot set {key!r} -> {value!r} on a future") class _FutureResponse(requests.Response): @@ -257,7 +255,7 @@ def finish(self): """ headers, body, timeout = self._prepare_batch_request() - url = "%s/batch/storage/v1" % self.API_BASE_URL + url = f"{self.API_BASE_URL}/batch/storage/v1" # Use the private ``_base_connection`` rather than the property # ``_connection``, since the property may be this @@ -332,7 +330,7 @@ def _unpack_batch_response(response): subresponse = requests.Response() subresponse.request = requests.Request( - method="BATCH", url="contentid://{}".format(content_id) + method="BATCH", url=f"contentid://{content_id}" ).prepare() subresponse.status_code = int(status) subresponse.headers.update(msg_headers) diff --git a/google/cloud/storage/blob.py b/google/cloud/storage/blob.py index 8a2b5861c..a3ea714ef 100644 --- a/google/cloud/storage/blob.py +++ b/google/cloud/storage/blob.py @@ -312,7 +312,7 @@ def __repr__(self): else: bucket_name = None - return "" % (bucket_name, self.name, self.generation) + return f"" @property def path(self): @@ -575,20 +575,16 @@ def generate_signed_url( quoted_name = _quote(self.name, safe=b"/~") if virtual_hosted_style: - api_access_endpoint = "https://{bucket_name}.storage.googleapis.com".format( - bucket_name=self.bucket.name - ) + api_access_endpoint = f"https://{self.bucket.name}.storage.googleapis.com" elif bucket_bound_hostname: api_access_endpoint = _bucket_bound_hostname_url( bucket_bound_hostname, scheme ) else: - resource = "/{bucket_name}/{quoted_name}".format( - bucket_name=self.bucket.name, quoted_name=quoted_name - ) + resource = f"/{self.bucket.name}/{quoted_name}" if virtual_hosted_style or bucket_bound_hostname: - resource = "/{quoted_name}".format(quoted_name=quoted_name) + resource = f"/{quoted_name}" if credentials is None: client = self._require_client(client) @@ -840,7 +836,7 @@ def _get_download_url( hostname = _get_host_name(client._connection) base_url = _DOWNLOAD_URL_TEMPLATE.format(hostname=hostname, path=self.path) if self.generation is not None: - name_value_pairs.append(("generation", "{:d}".format(self.generation))) + name_value_pairs.append(("generation", f"{self.generation:d}")) else: base_url = self.media_link @@ -3095,7 +3091,7 @@ def get_iam_policy( query_params["optionsRequestedPolicyVersion"] = requested_policy_version info = client._get_resource( - "%s/iam" % (self.path,), + f"{self.path}/iam", query_params=query_params, timeout=timeout, retry=retry, @@ -3151,7 +3147,7 @@ def set_iam_policy( if self.user_project is not None: query_params["userProject"] = self.user_project - path = "{}/iam".format(self.path) + path = f"{self.path}/iam" resource = policy.to_api_repr() resource["resourceId"] = self.path info = client._put_resource( @@ -3207,7 +3203,7 @@ def test_iam_permissions( if self.user_project is not None: query_params["userProject"] = self.user_project - path = "%s/iam/testPermissions" % (self.path,) + path = f"{self.path}/iam/testPermissions" resp = client._get_resource( path, query_params=query_params, @@ -3462,7 +3458,7 @@ def compose( ) api_response = client._post_resource( - "{}/compose".format(self.path), + f"{self.path}/compose", request, query_params=query_params, timeout=timeout, @@ -3595,7 +3591,7 @@ def rewrite( if_source_metageneration_not_match=if_source_metageneration_not_match, ) - path = "{}/rewriteTo{}".format(source.path, self.path) + path = f"{source.path}/rewriteTo{self.path}" api_response = client._post_resource( path, self._properties, @@ -3712,7 +3708,7 @@ def update_storage_class( (Optional) How to retry the RPC. See: :ref:`configuring_retries` """ if new_class not in self.STORAGE_CLASSES: - raise ValueError("Invalid storage class: %s" % (new_class,)) + raise ValueError(f"Invalid storage class: {new_class}") # Update current blob's storage class prior to rewrite self._patch_property("storageClass", new_class) @@ -3755,7 +3751,7 @@ def open( encoding=None, errors=None, newline=None, - **kwargs + **kwargs, ): r"""Create a file handler for file-like I/O to or from this blob. @@ -4448,9 +4444,7 @@ def _raise_from_invalid_response(error): else: error_message = str(error) - message = "{method} {url}: {error}".format( - method=response.request.method, url=response.request.url, error=error_message - ) + message = f"{response.request.method} {response.request.url}: {error_message}" raise exceptions.from_http_status(response.status_code, message, response=response) diff --git a/google/cloud/storage/bucket.py b/google/cloud/storage/bucket.py index be99ad141..143629236 100644 --- a/google/cloud/storage/bucket.py +++ b/google/cloud/storage/bucket.py @@ -647,7 +647,7 @@ def __init__(self, client, name=None, user_project=None): self._user_project = user_project def __repr__(self): - return "" % (self.name,) + return f"" @property def client(self): @@ -1166,7 +1166,7 @@ def get_blob( if_metageneration_not_match=None, timeout=_DEFAULT_TIMEOUT, retry=DEFAULT_RETRY, - **kwargs + **kwargs, ): """Get a blob object by name. @@ -1241,7 +1241,7 @@ def get_blob( name=blob_name, encryption_key=encryption_key, generation=generation, - **kwargs + **kwargs, ) try: # NOTE: This will not fail immediately in a batch. However, when @@ -2600,7 +2600,7 @@ def storage_class(self, value): :attr:`~google.cloud.storage.constants.DURABLE_REDUCED_AVAILABILITY_LEGACY_STORAGE_CLASS`, """ if value not in self.STORAGE_CLASSES: - raise ValueError("Invalid storage class: %s" % (value,)) + raise ValueError(f"Invalid storage class: {value}") self._patch_property("storageClass", value) @property @@ -2801,7 +2801,7 @@ def get_iam_policy( query_params["optionsRequestedPolicyVersion"] = requested_policy_version info = client._get_resource( - "%s/iam" % (self.path,), + f"{self.path}/iam", query_params=query_params, timeout=timeout, retry=retry, @@ -2850,7 +2850,7 @@ def set_iam_policy( if self.user_project is not None: query_params["userProject"] = self.user_project - path = "{}/iam".format(self.path) + path = f"{self.path}/iam" resource = policy.to_api_repr() resource["resourceId"] = self.path @@ -2902,7 +2902,7 @@ def test_iam_permissions( if self.user_project is not None: query_params["userProject"] = self.user_project - path = "%s/iam/testPermissions" % (self.path,) + path = f"{self.path}/iam/testPermissions" resp = client._get_resource( path, query_params=query_params, @@ -3207,7 +3207,7 @@ def lock_retention_policy( if self.user_project is not None: query_params["userProject"] = self.user_project - path = "/b/{}/lockRetentionPolicy".format(self.name) + path = f"/b/{self.name}/lockRetentionPolicy" api_response = client._post_resource( path, None, @@ -3341,15 +3341,13 @@ def generate_signed_url( raise ValueError("'version' must be either 'v2' or 'v4'") if virtual_hosted_style: - api_access_endpoint = "https://{bucket_name}.storage.googleapis.com".format( - bucket_name=self.name - ) + api_access_endpoint = f"https://{self.name}.storage.googleapis.com" elif bucket_bound_hostname: api_access_endpoint = _bucket_bound_hostname_url( bucket_bound_hostname, scheme ) else: - resource = "/{bucket_name}".format(bucket_name=self.name) + resource = f"/{self.name}" if virtual_hosted_style or bucket_bound_hostname: resource = "/" @@ -3389,6 +3387,4 @@ def _raise_if_len_differs(expected_len, **generation_match_args): """ for name, value in generation_match_args.items(): if value is not None and len(value) != expected_len: - raise ValueError( - "'{}' length must be the same as 'blobs' length".format(name) - ) + raise ValueError(f"'{name}' length must be the same as 'blobs' length") diff --git a/google/cloud/storage/client.py b/google/cloud/storage/client.py index 8b63a0198..f905e1be0 100644 --- a/google/cloud/storage/client.py +++ b/google/cloud/storage/client.py @@ -272,7 +272,7 @@ def get_service_account_email( if project is None: project = self.project - path = "/projects/%s/serviceAccount" % (project,) + path = f"/projects/{project}/serviceAccount" api_response = self._get_resource(path, timeout=timeout, retry=retry) return api_response["email_address"] @@ -1471,7 +1471,7 @@ def create_hmac_key( if project_id is None: project_id = self.project - path = "/projects/{}/hmacKeys".format(project_id) + path = f"/projects/{project_id}/hmacKeys" qs_params = {"serviceAccountEmail": service_account_email} if user_project is not None: @@ -1537,7 +1537,7 @@ def list_hmac_keys( if project_id is None: project_id = self.project - path = "/projects/{}/hmacKeys".format(project_id) + path = f"/projects/{project_id}/hmacKeys" extra_params = {} if service_account_email is not None: @@ -1747,11 +1747,11 @@ def generate_signed_post_policy_v4( ) # designate URL if virtual_hosted_style: - url = "https://{}.storage.googleapis.com/".format(bucket_name) + url = f"https://{bucket_name}.storage.googleapis.com/" elif bucket_bound_hostname: url = _bucket_bound_hostname_url(bucket_bound_hostname, scheme) else: - url = "https://storage.googleapis.com/{}/".format(bucket_name) + url = f"https://storage.googleapis.com/{bucket_name}/" return {"url": url, "fields": policy_fields} diff --git a/google/cloud/storage/fileio.py b/google/cloud/storage/fileio.py index e05663675..cc04800eb 100644 --- a/google/cloud/storage/fileio.py +++ b/google/cloud/storage/fileio.py @@ -106,7 +106,7 @@ def __init__(self, blob, chunk_size=None, retry=DEFAULT_RETRY, **download_kwargs for kwarg in download_kwargs: if kwarg not in VALID_DOWNLOAD_KWARGS: raise ValueError( - "BlobReader does not support keyword argument {}.".format(kwarg) + f"BlobReader does not support keyword argument {kwarg}." ) self._blob = blob @@ -144,7 +144,7 @@ def read(self, size=-1): end=fetch_end, checksum=None, retry=self._retry, - **self._download_kwargs + **self._download_kwargs, ) except RequestRangeNotSatisfiable: # We've reached the end of the file. Python file objects should @@ -299,12 +299,12 @@ def __init__( text_mode=False, ignore_flush=False, retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED, - **upload_kwargs + **upload_kwargs, ): for kwarg in upload_kwargs: if kwarg not in VALID_UPLOAD_KWARGS: raise ValueError( - "BlobWriter does not support keyword argument {}.".format(kwarg) + f"BlobWriter does not support keyword argument {kwarg}." ) self._blob = blob self._buffer = SlidingBuffer() @@ -390,7 +390,7 @@ def _initiate_upload(self): num_retries, chunk_size=self._chunk_size, retry=retry, - **self._upload_kwargs + **self._upload_kwargs, ) def _upload_chunks_from_buffer(self, num_chunks): diff --git a/google/cloud/storage/hmac_key.py b/google/cloud/storage/hmac_key.py index 1636aaba4..944bc7f87 100644 --- a/google/cloud/storage/hmac_key.py +++ b/google/cloud/storage/hmac_key.py @@ -133,9 +133,7 @@ def state(self): def state(self, value): if value not in self._SETTABLE_STATES: raise ValueError( - "State may only be set to one of: {}".format( - ", ".join(self._SETTABLE_STATES) - ) + f"State may only be set to one of: {', '.join(self._SETTABLE_STATES)}" ) self._properties["state"] = value @@ -177,7 +175,7 @@ def path(self): if project is None: project = self._client.project - return "/projects/{}/hmacKeys/{}".format(project, self.access_id) + return f"/projects/{project}/hmacKeys/{self.access_id}" @property def user_project(self): diff --git a/google/cloud/storage/notification.py b/google/cloud/storage/notification.py index 0cdb87fa8..f7e72e710 100644 --- a/google/cloud/storage/notification.py +++ b/google/cloud/storage/notification.py @@ -202,9 +202,7 @@ def client(self): @property def path(self): """The URL path for this notification.""" - return "/b/{}/notificationConfigs/{}".format( - self.bucket.name, self.notification_id - ) + return f"/b/{self.bucket.name}/notificationConfigs/{self.notification_id}" def _require_client(self, client): """Check client or verify over-ride. @@ -254,7 +252,7 @@ def create(self, client=None, timeout=_DEFAULT_TIMEOUT, retry=None): """ if self.notification_id is not None: raise ValueError( - "Notification already exists w/ id: {}".format(self.notification_id) + f"Notification already exists w/ id: {self.notification_id}" ) client = self._require_client(client) @@ -263,7 +261,7 @@ def create(self, client=None, timeout=_DEFAULT_TIMEOUT, retry=None): if self.bucket.user_project is not None: query_params["userProject"] = self.bucket.user_project - path = "/b/{}/notificationConfigs".format(self.bucket.name) + path = f"/b/{self.bucket.name}/notificationConfigs" properties = self._properties.copy() if self.topic_name is None: diff --git a/samples/snippets/acl_test.py b/samples/snippets/acl_test.py index 91856d816..eecee522b 100644 --- a/samples/snippets/acl_test.py +++ b/samples/snippets/acl_test.py @@ -46,7 +46,7 @@ def test_bucket(): os.environ["GOOGLE_CLOUD_PROJECT"] = os.environ["MAIN_GOOGLE_CLOUD_PROJECT"] bucket = None while bucket is None or bucket.exists(): - bucket_name = "acl-test-{}".format(uuid.uuid4()) + bucket_name = f"acl-test-{uuid.uuid4()}" bucket = storage.Client().bucket(bucket_name) bucket.create() yield bucket @@ -59,7 +59,7 @@ def test_bucket(): def test_blob(test_bucket): """Yields a blob that is deleted after the test completes.""" bucket = test_bucket - blob = bucket.blob("storage_acl_test_sigil-{}".format(uuid.uuid4())) + blob = bucket.blob(f"storage_acl_test_sigil-{uuid.uuid4()}") blob.upload_from_string("Hello, is it me you're looking for?") yield blob diff --git a/samples/snippets/bucket_lock_test.py b/samples/snippets/bucket_lock_test.py index 67d4ec685..9b7b4fa2a 100644 --- a/samples/snippets/bucket_lock_test.py +++ b/samples/snippets/bucket_lock_test.py @@ -42,7 +42,7 @@ def bucket(): """Yields a bucket that is deleted after the test completes.""" bucket = None while bucket is None or bucket.exists(): - bucket_name = "bucket-lock-{}".format(uuid.uuid4()) + bucket_name = f"bucket-lock-{uuid.uuid4()}" bucket = storage.Client().bucket(bucket_name) bucket.create() yield bucket @@ -61,7 +61,7 @@ def test_retention_policy_no_lock(bucket, capsys): storage_get_retention_policy.get_retention_policy(bucket.name) out, _ = capsys.readouterr() - assert "Retention Policy for {}".format(bucket.name) in out + assert f"Retention Policy for {bucket.name}" in out assert "Retention Period: 5" in out assert "Effective Time: " in out assert "Retention Policy is locked" not in out @@ -100,11 +100,11 @@ def test_enable_disable_bucket_default_event_based_hold(bucket, capsys): ) out, _ = capsys.readouterr() assert ( - "Default event-based hold is not enabled for {}".format(bucket.name) + f"Default event-based hold is not enabled for {bucket.name}" in out ) assert ( - "Default event-based hold is enabled for {}".format(bucket.name) + f"Default event-based hold is enabled for {bucket.name}" not in out ) @@ -120,7 +120,7 @@ def test_enable_disable_bucket_default_event_based_hold(bucket, capsys): ) out, _ = capsys.readouterr() assert ( - "Default event-based hold is enabled for {}".format(bucket.name) in out + f"Default event-based hold is enabled for {bucket.name}" in out ) # Changes to the bucket will be readable immediately after writing, diff --git a/samples/snippets/encryption_test.py b/samples/snippets/encryption_test.py index 6c2377e0f..536c5d334 100644 --- a/samples/snippets/encryption_test.py +++ b/samples/snippets/encryption_test.py @@ -62,7 +62,7 @@ def test_upload_encrypted_blob(): def test_blob(): """Provides a pre-existing blob in the test bucket.""" bucket = storage.Client().bucket(BUCKET) - blob_name = "test_blob_{}".format(uuid.uuid4().hex) + blob_name = f"test_blob_{uuid.uuid4().hex}" blob = Blob( blob_name, bucket, @@ -81,7 +81,7 @@ def test_blob(): blob.delete() except NotFound as e: # For the case that the rotation succeeded. - print("Ignoring 404, detail: {}".format(e)) + print(f"Ignoring 404, detail: {e}") blob = Blob( blob_name, bucket, diff --git a/samples/snippets/fileio_test.py b/samples/snippets/fileio_test.py index cf98ce1ab..b8a4b8272 100644 --- a/samples/snippets/fileio_test.py +++ b/samples/snippets/fileio_test.py @@ -19,14 +19,14 @@ def test_fileio_write_read(bucket, capsys): - blob_name = "test-fileio-{}".format(uuid.uuid4()) + blob_name = f"test-fileio-{uuid.uuid4()}" storage_fileio_write_read.write_read(bucket.name, blob_name) out, _ = capsys.readouterr() assert "Hello world" in out def test_fileio_pandas(bucket, capsys): - blob_name = "test-fileio-{}".format(uuid.uuid4()) + blob_name = f"test-fileio-{uuid.uuid4()}" storage_fileio_pandas.pandas_write(bucket.name, blob_name) out, _ = capsys.readouterr() assert f"Wrote csv with pandas with name {blob_name} from bucket {bucket.name}." in out diff --git a/samples/snippets/iam_test.py b/samples/snippets/iam_test.py index edeb8427d..7700b6c6a 100644 --- a/samples/snippets/iam_test.py +++ b/samples/snippets/iam_test.py @@ -42,7 +42,7 @@ def bucket(): bucket = None while bucket is None or bucket.exists(): storage_client = storage.Client() - bucket_name = "test-iam-{}".format(uuid.uuid4()) + bucket_name = f"test-iam-{uuid.uuid4()}" bucket = storage_client.bucket(bucket_name) bucket.iam_configuration.uniform_bucket_level_access_enabled = True storage_client.create_bucket(bucket) @@ -60,7 +60,7 @@ def public_bucket(): bucket = None while bucket is None or bucket.exists(): storage_client = storage.Client() - bucket_name = "test-iam-{}".format(uuid.uuid4()) + bucket_name = f"test-iam-{uuid.uuid4()}" bucket = storage_client.bucket(bucket_name) bucket.iam_configuration.uniform_bucket_level_access_enabled = True storage_client.create_bucket(bucket) diff --git a/samples/snippets/notification_polling.py b/samples/snippets/notification_polling.py index 34fd8cc3e..2ee6789c3 100644 --- a/samples/snippets/notification_polling.py +++ b/samples/snippets/notification_polling.py @@ -76,13 +76,9 @@ def summarize(message): ) if "overwroteGeneration" in attributes: - description += "\tOverwrote generation: %s\n" % ( - attributes["overwroteGeneration"] - ) + description += f"\tOverwrote generation: {attributes['overwroteGeneration']}\n" if "overwrittenByGeneration" in attributes: - description += "\tOverwritten by generation: %s\n" % ( - attributes["overwrittenByGeneration"] - ) + description += f"\tOverwritten by generation: {attributes['overwrittenByGeneration']}\n" payload_format = attributes["payloadFormat"] if payload_format == "JSON_API_V1": @@ -110,14 +106,14 @@ def poll_notifications(project, subscription_name): ) def callback(message): - print("Received message:\n{}".format(summarize(message))) + print(f"Received message:\n{summarize(message)}") message.ack() subscriber.subscribe(subscription_path, callback=callback) # The subscriber is non-blocking, so we must keep the main thread from # exiting to allow it to process messages in the background. - print("Listening for messages on {}".format(subscription_path)) + print(f"Listening for messages on {subscription_path}") while True: time.sleep(60) diff --git a/samples/snippets/notification_test.py b/samples/snippets/notification_test.py index 13553c844..a2fdbe3ef 100644 --- a/samples/snippets/notification_test.py +++ b/samples/snippets/notification_test.py @@ -55,7 +55,7 @@ def _notification_topic(storage_client, publisher_client): binding = policy.bindings.add() binding.role = "roles/pubsub.publisher" binding.members.append( - "serviceAccount:{}".format(storage_client.get_service_account_email()) + f"serviceAccount:{storage_client.get_service_account_email()}" ) publisher_client.set_iam_policy(request={"resource": topic_path, "policy": policy}) diff --git a/samples/snippets/noxfile.py b/samples/snippets/noxfile.py index 38bb0a572..9f1cc8fb1 100644 --- a/samples/snippets/noxfile.py +++ b/samples/snippets/noxfile.py @@ -66,7 +66,7 @@ sys.path.append(".") from noxfile_config import TEST_CONFIG_OVERRIDE except ImportError as e: - print("No user noxfile_config found: detail: {}".format(e)) + print(f"No user noxfile_config found: detail: {e}") TEST_CONFIG_OVERRIDE = {} # Update the TEST_CONFIG with the user supplied values. @@ -266,7 +266,7 @@ def py(session: nox.sessions.Session) -> None: _session_tests(session) else: session.skip( - "SKIPPED: {} tests are disabled for this sample.".format(session.python) + f"SKIPPED: {session.python} tests are disabled for this sample." ) diff --git a/samples/snippets/quickstart.py b/samples/snippets/quickstart.py index 578e50753..54148b1fb 100644 --- a/samples/snippets/quickstart.py +++ b/samples/snippets/quickstart.py @@ -29,7 +29,7 @@ def run_quickstart(): # Creates the new bucket bucket = storage_client.create_bucket(bucket_name) - print("Bucket {} created.".format(bucket.name)) + print(f"Bucket {bucket.name} created.") # [END storage_quickstart] diff --git a/samples/snippets/requester_pays_test.py b/samples/snippets/requester_pays_test.py index 9a178edb0..cf8c2d097 100644 --- a/samples/snippets/requester_pays_test.py +++ b/samples/snippets/requester_pays_test.py @@ -34,19 +34,19 @@ def test_enable_requester_pays(capsys): storage_enable_requester_pays.enable_requester_pays(BUCKET) out, _ = capsys.readouterr() - assert "Requester Pays has been enabled for {}".format(BUCKET) in out + assert f"Requester Pays has been enabled for {BUCKET}" in out def test_disable_requester_pays(capsys): storage_disable_requester_pays.disable_requester_pays(BUCKET) out, _ = capsys.readouterr() - assert "Requester Pays has been disabled for {}".format(BUCKET) in out + assert f"Requester Pays has been disabled for {BUCKET}" in out def test_get_requester_pays_status(capsys): storage_get_requester_pays_status.get_requester_pays_status(BUCKET) out, _ = capsys.readouterr() - assert "Requester Pays is disabled for {}".format(BUCKET) in out + assert f"Requester Pays is disabled for {BUCKET}" in out @pytest.fixture diff --git a/samples/snippets/rpo_test.py b/samples/snippets/rpo_test.py index d084710a9..f1f16e7fb 100644 --- a/samples/snippets/rpo_test.py +++ b/samples/snippets/rpo_test.py @@ -28,7 +28,7 @@ def dual_region_bucket(): """Yields a dual region bucket that is deleted after the test completes.""" bucket = None while bucket is None or bucket.exists(): - bucket_name = "bucket-lock-{}".format(uuid.uuid4()) + bucket_name = f"bucket-lock-{uuid.uuid4()}" bucket = storage.Client().bucket(bucket_name) bucket.location = "NAM4" bucket.create() @@ -55,7 +55,7 @@ def test_set_rpo_default(dual_region_bucket, capsys): def test_create_bucket_turbo_replication(capsys): - bucket_name = "test-rpo-{}".format(uuid.uuid4()) + bucket_name = f"test-rpo-{uuid.uuid4()}" storage_create_bucket_turbo_replication.create_bucket_turbo_replication(bucket_name) out, _ = capsys.readouterr() assert f"{bucket_name} created with RPO ASYNC_TURBO in NAM4." in out diff --git a/samples/snippets/snippets_test.py b/samples/snippets/snippets_test.py index 7a5a3a64f..bdd8c528e 100644 --- a/samples/snippets/snippets_test.py +++ b/samples/snippets/snippets_test.py @@ -111,7 +111,7 @@ def test_bucket(): """Yields a bucket that is deleted after the test completes.""" bucket = None while bucket is None or bucket.exists(): - bucket_name = "storage-snippets-test-{}".format(uuid.uuid4()) + bucket_name = f"storage-snippets-test-{uuid.uuid4()}" bucket = storage.Client().bucket(bucket_name) bucket.create() yield bucket @@ -127,7 +127,7 @@ def test_public_bucket(): bucket = None while bucket is None or bucket.exists(): storage_client = storage.Client() - bucket_name = "storage-snippets-test-{}".format(uuid.uuid4()) + bucket_name = f"storage-snippets-test-{uuid.uuid4()}" bucket = storage_client.bucket(bucket_name) storage_client.create_bucket(bucket) yield bucket @@ -140,7 +140,7 @@ def test_public_bucket(): def test_blob(test_bucket): """Yields a blob that is deleted after the test completes.""" bucket = test_bucket - blob = bucket.blob("storage_snippets_test_sigil-{}".format(uuid.uuid4())) + blob = bucket.blob(f"storage_snippets_test_sigil-{uuid.uuid4()}") blob.upload_from_string("Hello, is it me you're looking for?") yield blob @@ -149,7 +149,7 @@ def test_blob(test_bucket): def test_public_blob(test_public_bucket): """Yields a blob that is deleted after the test completes.""" bucket = test_public_bucket - blob = bucket.blob("storage_snippets_test_sigil-{}".format(uuid.uuid4())) + blob = bucket.blob(f"storage_snippets_test_sigil-{uuid.uuid4()}") blob.upload_from_string("Hello, is it me you're looking for?") yield blob @@ -159,7 +159,7 @@ def test_bucket_create(): """Yields a bucket object that is deleted after the test completes.""" bucket = None while bucket is None or bucket.exists(): - bucket_name = "storage-snippets-test-{}".format(uuid.uuid4()) + bucket_name = f"storage-snippets-test-{uuid.uuid4()}" bucket = storage.Client().bucket(bucket_name) yield bucket bucket.delete(force=True) @@ -217,7 +217,7 @@ def test_upload_blob_from_stream(test_bucket, capsys): ) out, _ = capsys.readouterr() - assert "Stream data uploaded to {}".format("test_upload_blob") in out + assert "Stream data uploaded to test_upload_blob" in out def test_upload_blob_with_kms(test_bucket): @@ -339,7 +339,7 @@ def test_generate_signed_policy_v4(test_bucket, capsys): blob_name = "storage_snippets_test_form" short_name = storage_generate_signed_post_policy_v4 form = short_name.generate_signed_post_policy_v4(test_bucket.name, blob_name) - assert "name='key' value='{}'".format(blob_name) in form + assert f"name='key' value='{blob_name}'" in form assert "name='x-goog-signature'" in form assert "name='x-goog-date'" in form assert "name='x-goog-credential'" in form @@ -355,7 +355,7 @@ def test_rename_blob(test_blob): try: bucket.delete_blob("test_rename_blob") except google.cloud.exceptions.exceptions.NotFound: - print("test_rename_blob not found in bucket {}".format(bucket.name)) + print(f"test_rename_blob not found in bucket {bucket.name}") storage_rename_file.rename_blob(bucket.name, test_blob.name, "test_rename_blob") @@ -370,7 +370,7 @@ def test_move_blob(test_bucket_create, test_blob): try: test_bucket_create.delete_blob("test_move_blob") except google.cloud.exceptions.NotFound: - print("test_move_blob not found in bucket {}".format(test_bucket_create.name)) + print(f"test_move_blob not found in bucket {test_bucket_create.name}") storage_move_file.move_blob( bucket.name, test_blob.name, test_bucket_create.name, "test_move_blob" @@ -551,7 +551,7 @@ def test_change_file_storage_class(test_blob, capsys): test_blob.bucket.name, test_blob.name ) out, _ = capsys.readouterr() - assert "Blob {} in bucket {}". format(blob.name, blob.bucket.name) in out + assert f"Blob {blob.name} in bucket {blob.bucket.name}" in out assert blob.storage_class == 'NEARLINE' diff --git a/samples/snippets/storage_activate_hmac_key.py b/samples/snippets/storage_activate_hmac_key.py index e77cd8066..d3960eb62 100644 --- a/samples/snippets/storage_activate_hmac_key.py +++ b/samples/snippets/storage_activate_hmac_key.py @@ -36,14 +36,14 @@ def activate_key(access_id, project_id): hmac_key.update() print("The HMAC key metadata is:") - print("Service Account Email: {}".format(hmac_key.service_account_email)) - print("Key ID: {}".format(hmac_key.id)) - print("Access ID: {}".format(hmac_key.access_id)) - print("Project ID: {}".format(hmac_key.project)) - print("State: {}".format(hmac_key.state)) - print("Created At: {}".format(hmac_key.time_created)) - print("Updated At: {}".format(hmac_key.updated)) - print("Etag: {}".format(hmac_key.etag)) + print(f"Service Account Email: {hmac_key.service_account_email}") + print(f"Key ID: {hmac_key.id}") + print(f"Access ID: {hmac_key.access_id}") + print(f"Project ID: {hmac_key.project}") + print(f"State: {hmac_key.state}") + print(f"Created At: {hmac_key.time_created}") + print(f"Updated At: {hmac_key.updated}") + print(f"Etag: {hmac_key.etag}") return hmac_key diff --git a/samples/snippets/storage_add_bucket_conditional_iam_binding.py b/samples/snippets/storage_add_bucket_conditional_iam_binding.py index ddc0fc028..d09f528cf 100644 --- a/samples/snippets/storage_add_bucket_conditional_iam_binding.py +++ b/samples/snippets/storage_add_bucket_conditional_iam_binding.py @@ -53,15 +53,15 @@ def add_bucket_conditional_iam_binding( bucket.set_iam_policy(policy) - print("Added the following member(s) with role {} to {}:".format(role, bucket_name)) + print(f"Added the following member(s) with role {role} to {bucket_name}:") for member in members: - print(" {}".format(member)) + print(f" {member}") print("with condition:") - print(" Title: {}".format(title)) - print(" Description: {}".format(description)) - print(" Expression: {}".format(expression)) + print(f" Title: {title}") + print(f" Description: {description}") + print(f" Expression: {expression}") # [END storage_add_bucket_conditional_iam_binding] diff --git a/samples/snippets/storage_add_bucket_iam_member.py b/samples/snippets/storage_add_bucket_iam_member.py index 727f18483..0d610eae7 100644 --- a/samples/snippets/storage_add_bucket_iam_member.py +++ b/samples/snippets/storage_add_bucket_iam_member.py @@ -35,7 +35,7 @@ def add_bucket_iam_member(bucket_name, role, member): bucket.set_iam_policy(policy) - print("Added {} with role {} to {}.".format(member, role, bucket_name)) + print(f"Added {member} with role {role} to {bucket_name}.") # [END storage_add_bucket_iam_member] diff --git a/samples/snippets/storage_add_bucket_label.py b/samples/snippets/storage_add_bucket_label.py index 8ae8fe1f4..9c6fcff7a 100644 --- a/samples/snippets/storage_add_bucket_label.py +++ b/samples/snippets/storage_add_bucket_label.py @@ -36,7 +36,7 @@ def add_bucket_label(bucket_name): bucket.labels = labels bucket.patch() - print("Updated labels on {}.".format(bucket.name)) + print(f"Updated labels on {bucket.name}.") pprint.pprint(bucket.labels) diff --git a/samples/snippets/storage_add_bucket_owner.py b/samples/snippets/storage_add_bucket_owner.py index acdb60dc5..bac1f3f64 100644 --- a/samples/snippets/storage_add_bucket_owner.py +++ b/samples/snippets/storage_add_bucket_owner.py @@ -40,9 +40,7 @@ def add_bucket_owner(bucket_name, user_email): bucket.acl.save() print( - "Added user {} as an owner on bucket {}.".format( - user_email, bucket_name - ) + f"Added user {user_email} as an owner on bucket {bucket_name}." ) diff --git a/samples/snippets/storage_bucket_delete_default_kms_key.py b/samples/snippets/storage_bucket_delete_default_kms_key.py index 3df23767d..0db293756 100644 --- a/samples/snippets/storage_bucket_delete_default_kms_key.py +++ b/samples/snippets/storage_bucket_delete_default_kms_key.py @@ -30,7 +30,7 @@ def bucket_delete_default_kms_key(bucket_name): bucket.default_kms_key_name = None bucket.patch() - print("Default KMS key was removed from {}".format(bucket.name)) + print(f"Default KMS key was removed from {bucket.name}") return bucket diff --git a/samples/snippets/storage_change_default_storage_class.py b/samples/snippets/storage_change_default_storage_class.py index 8a72719ba..5d2f924ad 100644 --- a/samples/snippets/storage_change_default_storage_class.py +++ b/samples/snippets/storage_change_default_storage_class.py @@ -31,7 +31,7 @@ def change_default_storage_class(bucket_name): bucket.storage_class = constants.COLDLINE_STORAGE_CLASS bucket.patch() - print("Default storage class for bucket {} has been set to {}".format(bucket_name, bucket.storage_class)) + print(f"Default storage class for bucket {bucket_name} has been set to {bucket.storage_class}") return bucket diff --git a/samples/snippets/storage_configure_retries.py b/samples/snippets/storage_configure_retries.py index 9543111b3..ef1e422b6 100644 --- a/samples/snippets/storage_configure_retries.py +++ b/samples/snippets/storage_configure_retries.py @@ -53,7 +53,7 @@ def configure_retries(bucket_name, blob_name): ) blob.delete(retry=modified_retry) - print("Blob {} deleted with a customized retry strategy.".format(blob_name)) + print(f"Blob {blob_name} deleted with a customized retry strategy.") # [END storage_configure_retries] diff --git a/samples/snippets/storage_cors_configuration.py b/samples/snippets/storage_cors_configuration.py index 3d2595a9d..2c5dd2428 100644 --- a/samples/snippets/storage_cors_configuration.py +++ b/samples/snippets/storage_cors_configuration.py @@ -38,7 +38,7 @@ def cors_configuration(bucket_name): ] bucket.patch() - print("Set CORS policies for bucket {} is {}".format(bucket.name, bucket.cors)) + print(f"Set CORS policies for bucket {bucket.name} is {bucket.cors}") return bucket diff --git a/samples/snippets/storage_create_bucket.py b/samples/snippets/storage_create_bucket.py index aaee9e234..c95f32f56 100644 --- a/samples/snippets/storage_create_bucket.py +++ b/samples/snippets/storage_create_bucket.py @@ -28,7 +28,7 @@ def create_bucket(bucket_name): bucket = storage_client.create_bucket(bucket_name) - print("Bucket {} created".format(bucket.name)) + print(f"Bucket {bucket.name} created") # [END storage_create_bucket] diff --git a/samples/snippets/storage_create_hmac_key.py b/samples/snippets/storage_create_hmac_key.py index 27a418c39..d845738b7 100644 --- a/samples/snippets/storage_create_hmac_key.py +++ b/samples/snippets/storage_create_hmac_key.py @@ -33,17 +33,17 @@ def create_key(project_id, service_account_email): service_account_email=service_account_email, project_id=project_id ) - print("The base64 encoded secret is {}".format(secret)) + print(f"The base64 encoded secret is {secret}") print("Do not miss that secret, there is no API to recover it.") print("The HMAC key metadata is:") - print("Service Account Email: {}".format(hmac_key.service_account_email)) - print("Key ID: {}".format(hmac_key.id)) - print("Access ID: {}".format(hmac_key.access_id)) - print("Project ID: {}".format(hmac_key.project)) - print("State: {}".format(hmac_key.state)) - print("Created At: {}".format(hmac_key.time_created)) - print("Updated At: {}".format(hmac_key.updated)) - print("Etag: {}".format(hmac_key.etag)) + print(f"Service Account Email: {hmac_key.service_account_email}") + print(f"Key ID: {hmac_key.id}") + print(f"Access ID: {hmac_key.access_id}") + print(f"Project ID: {hmac_key.project}") + print(f"State: {hmac_key.state}") + print(f"Created At: {hmac_key.time_created}") + print(f"Updated At: {hmac_key.updated}") + print(f"Etag: {hmac_key.etag}") return hmac_key diff --git a/samples/snippets/storage_deactivate_hmac_key.py b/samples/snippets/storage_deactivate_hmac_key.py index 389efb998..007f7b5a5 100644 --- a/samples/snippets/storage_deactivate_hmac_key.py +++ b/samples/snippets/storage_deactivate_hmac_key.py @@ -37,14 +37,14 @@ def deactivate_key(access_id, project_id): print("The HMAC key is now inactive.") print("The HMAC key metadata is:") - print("Service Account Email: {}".format(hmac_key.service_account_email)) - print("Key ID: {}".format(hmac_key.id)) - print("Access ID: {}".format(hmac_key.access_id)) - print("Project ID: {}".format(hmac_key.project)) - print("State: {}".format(hmac_key.state)) - print("Created At: {}".format(hmac_key.time_created)) - print("Updated At: {}".format(hmac_key.updated)) - print("Etag: {}".format(hmac_key.etag)) + print(f"Service Account Email: {hmac_key.service_account_email}") + print(f"Key ID: {hmac_key.id}") + print(f"Access ID: {hmac_key.access_id}") + print(f"Project ID: {hmac_key.project}") + print(f"State: {hmac_key.state}") + print(f"Created At: {hmac_key.time_created}") + print(f"Updated At: {hmac_key.updated}") + print(f"Etag: {hmac_key.etag}") return hmac_key diff --git a/samples/snippets/storage_delete_bucket.py b/samples/snippets/storage_delete_bucket.py index b3e264c74..b12c06636 100644 --- a/samples/snippets/storage_delete_bucket.py +++ b/samples/snippets/storage_delete_bucket.py @@ -29,7 +29,7 @@ def delete_bucket(bucket_name): bucket = storage_client.get_bucket(bucket_name) bucket.delete() - print("Bucket {} deleted".format(bucket.name)) + print(f"Bucket {bucket.name} deleted") # [END storage_delete_bucket] diff --git a/samples/snippets/storage_delete_file.py b/samples/snippets/storage_delete_file.py index 1105f3725..b2997c86b 100644 --- a/samples/snippets/storage_delete_file.py +++ b/samples/snippets/storage_delete_file.py @@ -31,7 +31,7 @@ def delete_blob(bucket_name, blob_name): blob = bucket.blob(blob_name) blob.delete() - print("Blob {} deleted.".format(blob_name)) + print(f"Blob {blob_name} deleted.") # [END storage_delete_file] diff --git a/samples/snippets/storage_delete_file_archived_generation.py b/samples/snippets/storage_delete_file_archived_generation.py index 4e4909001..ff02bca23 100644 --- a/samples/snippets/storage_delete_file_archived_generation.py +++ b/samples/snippets/storage_delete_file_archived_generation.py @@ -31,9 +31,7 @@ def delete_file_archived_generation(bucket_name, blob_name, generation): bucket = storage_client.get_bucket(bucket_name) bucket.delete_blob(blob_name, generation=generation) print( - "Generation {} of blob {} was deleted from {}".format( - generation, blob_name, bucket_name - ) + f"Generation {generation} of blob {blob_name} was deleted from {bucket_name}" ) diff --git a/samples/snippets/storage_disable_bucket_lifecycle_management.py b/samples/snippets/storage_disable_bucket_lifecycle_management.py index 9ef6971fb..a5fa56fcf 100644 --- a/samples/snippets/storage_disable_bucket_lifecycle_management.py +++ b/samples/snippets/storage_disable_bucket_lifecycle_management.py @@ -31,7 +31,7 @@ def disable_bucket_lifecycle_management(bucket_name): bucket.patch() rules = bucket.lifecycle_rules - print("Lifecycle management is disable for bucket {} and the rules are {}".format(bucket_name, list(rules))) + print(f"Lifecycle management is disable for bucket {bucket_name} and the rules are {list(rules)}") return bucket diff --git a/samples/snippets/storage_disable_default_event_based_hold.py b/samples/snippets/storage_disable_default_event_based_hold.py index dff3ed3c1..48becdac1 100644 --- a/samples/snippets/storage_disable_default_event_based_hold.py +++ b/samples/snippets/storage_disable_default_event_based_hold.py @@ -30,7 +30,7 @@ def disable_default_event_based_hold(bucket_name): bucket.default_event_based_hold = False bucket.patch() - print("Default event based hold was disabled for {}".format(bucket_name)) + print(f"Default event based hold was disabled for {bucket_name}") # [END storage_disable_default_event_based_hold] diff --git a/samples/snippets/storage_disable_requester_pays.py b/samples/snippets/storage_disable_requester_pays.py index c49cc28ea..78e195d8a 100644 --- a/samples/snippets/storage_disable_requester_pays.py +++ b/samples/snippets/storage_disable_requester_pays.py @@ -30,7 +30,7 @@ def disable_requester_pays(bucket_name): bucket.requester_pays = False bucket.patch() - print("Requester Pays has been disabled for {}".format(bucket_name)) + print(f"Requester Pays has been disabled for {bucket_name}") # [END storage_disable_requester_pays] diff --git a/samples/snippets/storage_disable_uniform_bucket_level_access.py b/samples/snippets/storage_disable_uniform_bucket_level_access.py index 4f4691611..20a045686 100644 --- a/samples/snippets/storage_disable_uniform_bucket_level_access.py +++ b/samples/snippets/storage_disable_uniform_bucket_level_access.py @@ -31,7 +31,7 @@ def disable_uniform_bucket_level_access(bucket_name): bucket.patch() print( - "Uniform bucket-level access was disabled for {}.".format(bucket.name) + f"Uniform bucket-level access was disabled for {bucket.name}." ) diff --git a/samples/snippets/storage_disable_versioning.py b/samples/snippets/storage_disable_versioning.py index 98832ba68..9dfd0ff90 100644 --- a/samples/snippets/storage_disable_versioning.py +++ b/samples/snippets/storage_disable_versioning.py @@ -30,7 +30,7 @@ def disable_versioning(bucket_name): bucket.versioning_enabled = False bucket.patch() - print("Versioning was disabled for bucket {}".format(bucket)) + print(f"Versioning was disabled for bucket {bucket}") return bucket diff --git a/samples/snippets/storage_download_encrypted_file.py b/samples/snippets/storage_download_encrypted_file.py index ac7071fbe..8a81b0de5 100644 --- a/samples/snippets/storage_download_encrypted_file.py +++ b/samples/snippets/storage_download_encrypted_file.py @@ -52,9 +52,7 @@ def download_encrypted_blob( blob.download_to_filename(destination_file_name) print( - "Blob {} downloaded to {}.".format( - source_blob_name, destination_file_name - ) + f"Blob {source_blob_name} downloaded to {destination_file_name}." ) diff --git a/samples/snippets/storage_download_to_stream.py b/samples/snippets/storage_download_to_stream.py index 1cb8dcc7b..3834e34c9 100644 --- a/samples/snippets/storage_download_to_stream.py +++ b/samples/snippets/storage_download_to_stream.py @@ -42,7 +42,7 @@ def download_blob_to_stream(bucket_name, source_blob_name, file_obj): blob = bucket.blob(source_blob_name) blob.download_to_file(file_obj) - print("Downloaded blob {} to file-like object.".format(source_blob_name)) + print(f"Downloaded blob {source_blob_name} to file-like object.") return file_obj # Before reading from file_obj, remember to rewind with file_obj.seek(0). diff --git a/samples/snippets/storage_enable_bucket_lifecycle_management.py b/samples/snippets/storage_enable_bucket_lifecycle_management.py index 61c7d7b20..0bbff079c 100644 --- a/samples/snippets/storage_enable_bucket_lifecycle_management.py +++ b/samples/snippets/storage_enable_bucket_lifecycle_management.py @@ -29,12 +29,12 @@ def enable_bucket_lifecycle_management(bucket_name): bucket = storage_client.get_bucket(bucket_name) rules = bucket.lifecycle_rules - print("Lifecycle management rules for bucket {} are {}".format(bucket_name, list(rules))) + print(f"Lifecycle management rules for bucket {bucket_name} are {list(rules)}") bucket.add_lifecycle_delete_rule(age=2) bucket.patch() rules = bucket.lifecycle_rules - print("Lifecycle management is enable for bucket {} and the rules are {}".format(bucket_name, list(rules))) + print(f"Lifecycle management is enable for bucket {bucket_name} and the rules are {list(rules)}") return bucket diff --git a/samples/snippets/storage_enable_default_event_based_hold.py b/samples/snippets/storage_enable_default_event_based_hold.py index a535390c9..5dfdf94a9 100644 --- a/samples/snippets/storage_enable_default_event_based_hold.py +++ b/samples/snippets/storage_enable_default_event_based_hold.py @@ -30,7 +30,7 @@ def enable_default_event_based_hold(bucket_name): bucket.default_event_based_hold = True bucket.patch() - print("Default event based hold was enabled for {}".format(bucket_name)) + print(f"Default event based hold was enabled for {bucket_name}") # [END storage_enable_default_event_based_hold] diff --git a/samples/snippets/storage_enable_requester_pays.py b/samples/snippets/storage_enable_requester_pays.py index 9787008dd..fbecb04f4 100644 --- a/samples/snippets/storage_enable_requester_pays.py +++ b/samples/snippets/storage_enable_requester_pays.py @@ -30,7 +30,7 @@ def enable_requester_pays(bucket_name): bucket.requester_pays = True bucket.patch() - print("Requester Pays has been enabled for {}".format(bucket_name)) + print(f"Requester Pays has been enabled for {bucket_name}") # [END storage_enable_requester_pays] diff --git a/samples/snippets/storage_enable_uniform_bucket_level_access.py b/samples/snippets/storage_enable_uniform_bucket_level_access.py index c689bb735..9ab71ae37 100644 --- a/samples/snippets/storage_enable_uniform_bucket_level_access.py +++ b/samples/snippets/storage_enable_uniform_bucket_level_access.py @@ -31,7 +31,7 @@ def enable_uniform_bucket_level_access(bucket_name): bucket.patch() print( - "Uniform bucket-level access was enabled for {}.".format(bucket.name) + f"Uniform bucket-level access was enabled for {bucket.name}." ) diff --git a/samples/snippets/storage_enable_versioning.py b/samples/snippets/storage_enable_versioning.py index 89693e426..9cdc98001 100644 --- a/samples/snippets/storage_enable_versioning.py +++ b/samples/snippets/storage_enable_versioning.py @@ -30,7 +30,7 @@ def enable_versioning(bucket_name): bucket.versioning_enabled = True bucket.patch() - print("Versioning was enabled for bucket {}".format(bucket.name)) + print(f"Versioning was enabled for bucket {bucket.name}") return bucket diff --git a/samples/snippets/storage_generate_encryption_key.py b/samples/snippets/storage_generate_encryption_key.py index a973418a6..dbeb46b91 100644 --- a/samples/snippets/storage_generate_encryption_key.py +++ b/samples/snippets/storage_generate_encryption_key.py @@ -30,7 +30,7 @@ def generate_encryption_key(): key = os.urandom(32) encoded_key = base64.b64encode(key).decode("utf-8") - print("Base 64 encoded encryption key: {}".format(encoded_key)) + print(f"Base 64 encoded encryption key: {encoded_key}") # [END storage_generate_encryption_key] diff --git a/samples/snippets/storage_generate_signed_post_policy_v4.py b/samples/snippets/storage_generate_signed_post_policy_v4.py index 8217714e2..0c06ddc2f 100644 --- a/samples/snippets/storage_generate_signed_post_policy_v4.py +++ b/samples/snippets/storage_generate_signed_post_policy_v4.py @@ -46,7 +46,7 @@ def generate_signed_post_policy_v4(bucket_name, blob_name): # Include all fields returned in the HTML form as they're required for key, value in policy["fields"].items(): - form += " \n".format(key, value) + form += f" \n" form += "
\n" form += "
\n" diff --git a/samples/snippets/storage_generate_signed_url_v2.py b/samples/snippets/storage_generate_signed_url_v2.py index abea3dd54..f1317ea2f 100644 --- a/samples/snippets/storage_generate_signed_url_v2.py +++ b/samples/snippets/storage_generate_signed_url_v2.py @@ -44,7 +44,7 @@ def generate_signed_url(bucket_name, blob_name): method="GET", ) - print("The signed url for {} is {}".format(blob.name, url)) + print(f"The signed url for {blob.name} is {url}") return url diff --git a/samples/snippets/storage_generate_signed_url_v4.py b/samples/snippets/storage_generate_signed_url_v4.py index 2a45b23e9..80625a7b3 100644 --- a/samples/snippets/storage_generate_signed_url_v4.py +++ b/samples/snippets/storage_generate_signed_url_v4.py @@ -49,7 +49,7 @@ def generate_download_signed_url_v4(bucket_name, blob_name): print("Generated GET signed URL:") print(url) print("You can use this URL with any user agent, for example:") - print("curl '{}'".format(url)) + print(f"curl '{url}'") return url diff --git a/samples/snippets/storage_get_default_event_based_hold.py b/samples/snippets/storage_get_default_event_based_hold.py index 4cf13914d..08a05f8ef 100644 --- a/samples/snippets/storage_get_default_event_based_hold.py +++ b/samples/snippets/storage_get_default_event_based_hold.py @@ -29,12 +29,10 @@ def get_default_event_based_hold(bucket_name): bucket = storage_client.get_bucket(bucket_name) if bucket.default_event_based_hold: - print("Default event-based hold is enabled for {}".format(bucket_name)) + print(f"Default event-based hold is enabled for {bucket_name}") else: print( - "Default event-based hold is not enabled for {}".format( - bucket_name - ) + f"Default event-based hold is not enabled for {bucket_name}" ) diff --git a/samples/snippets/storage_get_hmac_key.py b/samples/snippets/storage_get_hmac_key.py index 4dc52240d..82b28ff99 100644 --- a/samples/snippets/storage_get_hmac_key.py +++ b/samples/snippets/storage_get_hmac_key.py @@ -34,14 +34,14 @@ def get_key(access_id, project_id): ) print("The HMAC key metadata is:") - print("Service Account Email: {}".format(hmac_key.service_account_email)) - print("Key ID: {}".format(hmac_key.id)) - print("Access ID: {}".format(hmac_key.access_id)) - print("Project ID: {}".format(hmac_key.project)) - print("State: {}".format(hmac_key.state)) - print("Created At: {}".format(hmac_key.time_created)) - print("Updated At: {}".format(hmac_key.updated)) - print("Etag: {}".format(hmac_key.etag)) + print(f"Service Account Email: {hmac_key.service_account_email}") + print(f"Key ID: {hmac_key.id}") + print(f"Access ID: {hmac_key.access_id}") + print(f"Project ID: {hmac_key.project}") + print(f"State: {hmac_key.state}") + print(f"Created At: {hmac_key.time_created}") + print(f"Updated At: {hmac_key.updated}") + print(f"Etag: {hmac_key.etag}") return hmac_key diff --git a/samples/snippets/storage_get_metadata.py b/samples/snippets/storage_get_metadata.py index 3ce7ecea8..eece8028a 100644 --- a/samples/snippets/storage_get_metadata.py +++ b/samples/snippets/storage_get_metadata.py @@ -33,27 +33,27 @@ def blob_metadata(bucket_name, blob_name): # make an HTTP request. blob = bucket.get_blob(blob_name) - print("Blob: {}".format(blob.name)) - print("Bucket: {}".format(blob.bucket.name)) - print("Storage class: {}".format(blob.storage_class)) - print("ID: {}".format(blob.id)) - print("Size: {} bytes".format(blob.size)) - print("Updated: {}".format(blob.updated)) - print("Generation: {}".format(blob.generation)) - print("Metageneration: {}".format(blob.metageneration)) - print("Etag: {}".format(blob.etag)) - print("Owner: {}".format(blob.owner)) - print("Component count: {}".format(blob.component_count)) - print("Crc32c: {}".format(blob.crc32c)) - print("md5_hash: {}".format(blob.md5_hash)) - print("Cache-control: {}".format(blob.cache_control)) - print("Content-type: {}".format(blob.content_type)) - print("Content-disposition: {}".format(blob.content_disposition)) - print("Content-encoding: {}".format(blob.content_encoding)) - print("Content-language: {}".format(blob.content_language)) - print("Metadata: {}".format(blob.metadata)) - print("Medialink: {}".format(blob.media_link)) - print("Custom Time: {}".format(blob.custom_time)) + print(f"Blob: {blob.name}") + print(f"Bucket: {blob.bucket.name}") + print(f"Storage class: {blob.storage_class}") + print(f"ID: {blob.id}") + print(f"Size: {blob.size} bytes") + print(f"Updated: {blob.updated}") + print(f"Generation: {blob.generation}") + print(f"Metageneration: {blob.metageneration}") + print(f"Etag: {blob.etag}") + print(f"Owner: {blob.owner}") + print(f"Component count: {blob.component_count}") + print(f"Crc32c: {blob.crc32c}") + print(f"md5_hash: {blob.md5_hash}") + print(f"Cache-control: {blob.cache_control}") + print(f"Content-type: {blob.content_type}") + print(f"Content-disposition: {blob.content_disposition}") + print(f"Content-encoding: {blob.content_encoding}") + print(f"Content-language: {blob.content_language}") + print(f"Metadata: {blob.metadata}") + print(f"Medialink: {blob.media_link}") + print(f"Custom Time: {blob.custom_time}") print("Temporary hold: ", "enabled" if blob.temporary_hold else "disabled") print( "Event based hold: ", @@ -61,9 +61,7 @@ def blob_metadata(bucket_name, blob_name): ) if blob.retention_expiration_time: print( - "retentionExpirationTime: {}".format( - blob.retention_expiration_time - ) + f"retentionExpirationTime: {blob.retention_expiration_time}" ) diff --git a/samples/snippets/storage_get_requester_pays_status.py b/samples/snippets/storage_get_requester_pays_status.py index 2014d654c..a2eeb34d7 100644 --- a/samples/snippets/storage_get_requester_pays_status.py +++ b/samples/snippets/storage_get_requester_pays_status.py @@ -29,9 +29,9 @@ def get_requester_pays_status(bucket_name): requester_pays_status = bucket.requester_pays if requester_pays_status: - print("Requester Pays is enabled for {}".format(bucket_name)) + print(f"Requester Pays is enabled for {bucket_name}") else: - print("Requester Pays is disabled for {}".format(bucket_name)) + print(f"Requester Pays is disabled for {bucket_name}") # [END storage_get_requester_pays_status] diff --git a/samples/snippets/storage_get_retention_policy.py b/samples/snippets/storage_get_retention_policy.py index f2ca26d26..215f80d5a 100644 --- a/samples/snippets/storage_get_retention_policy.py +++ b/samples/snippets/storage_get_retention_policy.py @@ -28,14 +28,14 @@ def get_retention_policy(bucket_name): bucket = storage_client.bucket(bucket_name) bucket.reload() - print("Retention Policy for {}".format(bucket_name)) - print("Retention Period: {}".format(bucket.retention_period)) + print(f"Retention Policy for {bucket_name}") + print(f"Retention Period: {bucket.retention_period}") if bucket.retention_policy_locked: print("Retention Policy is locked") if bucket.retention_policy_effective_time: print( - "Effective Time: {}".format(bucket.retention_policy_effective_time) + f"Effective Time: {bucket.retention_policy_effective_time}" ) diff --git a/samples/snippets/storage_get_service_account.py b/samples/snippets/storage_get_service_account.py index 58ababb91..5ac0e5638 100644 --- a/samples/snippets/storage_get_service_account.py +++ b/samples/snippets/storage_get_service_account.py @@ -25,9 +25,7 @@ def get_service_account(): email = storage_client.get_service_account_email() print( - "The GCS service account for project {} is: {} ".format( - storage_client.project, email - ) + f"The GCS service account for project {storage_client.project} is: {email} " ) diff --git a/samples/snippets/storage_get_uniform_bucket_level_access.py b/samples/snippets/storage_get_uniform_bucket_level_access.py index eddb8bc1a..206b9f1ff 100644 --- a/samples/snippets/storage_get_uniform_bucket_level_access.py +++ b/samples/snippets/storage_get_uniform_bucket_level_access.py @@ -30,9 +30,7 @@ def get_uniform_bucket_level_access(bucket_name): if iam_configuration.uniform_bucket_level_access_enabled: print( - "Uniform bucket-level access is enabled for {}.".format( - bucket.name - ) + f"Uniform bucket-level access is enabled for {bucket.name}." ) print( "Bucket will be locked on {}.".format( @@ -41,9 +39,7 @@ def get_uniform_bucket_level_access(bucket_name): ) else: print( - "Uniform bucket-level access is disabled for {}.".format( - bucket.name - ) + f"Uniform bucket-level access is disabled for {bucket.name}." ) diff --git a/samples/snippets/storage_list_file_archived_generations.py b/samples/snippets/storage_list_file_archived_generations.py index dc2f5eaf5..419cc3da4 100644 --- a/samples/snippets/storage_list_file_archived_generations.py +++ b/samples/snippets/storage_list_file_archived_generations.py @@ -29,7 +29,7 @@ def list_file_archived_generations(bucket_name): blobs = storage_client.list_blobs(bucket_name, versions=True) for blob in blobs: - print("{},{}".format(blob.name, blob.generation)) + print(f"{blob.name},{blob.generation}") # [END storage_list_file_archived_generations] diff --git a/samples/snippets/storage_list_hmac_keys.py b/samples/snippets/storage_list_hmac_keys.py index 8e5c53b58..a09616fa5 100644 --- a/samples/snippets/storage_list_hmac_keys.py +++ b/samples/snippets/storage_list_hmac_keys.py @@ -31,9 +31,9 @@ def list_keys(project_id): print("HMAC Keys:") for hmac_key in hmac_keys: print( - "Service Account Email: {}".format(hmac_key.service_account_email) + f"Service Account Email: {hmac_key.service_account_email}" ) - print("Access ID: {}".format(hmac_key.access_id)) + print(f"Access ID: {hmac_key.access_id}") return hmac_keys diff --git a/samples/snippets/storage_lock_retention_policy.py b/samples/snippets/storage_lock_retention_policy.py index d59572f5d..adff364d7 100644 --- a/samples/snippets/storage_lock_retention_policy.py +++ b/samples/snippets/storage_lock_retention_policy.py @@ -33,11 +33,9 @@ def lock_retention_policy(bucket_name): # and retention period can only be increased. bucket.lock_retention_policy() - print("Retention policy for {} is now locked".format(bucket_name)) + print(f"Retention policy for {bucket_name} is now locked") print( - "Retention policy effective as of {}".format( - bucket.retention_policy_effective_time - ) + f"Retention policy effective as of {bucket.retention_policy_effective_time}" ) diff --git a/samples/snippets/storage_make_public.py b/samples/snippets/storage_make_public.py index 79ae40d12..489508cf6 100644 --- a/samples/snippets/storage_make_public.py +++ b/samples/snippets/storage_make_public.py @@ -32,9 +32,7 @@ def make_blob_public(bucket_name, blob_name): blob.make_public() print( - "Blob {} is publicly accessible at {}".format( - blob.name, blob.public_url - ) + f"Blob {blob.name} is publicly accessible at {blob.public_url}" ) diff --git a/samples/snippets/storage_object_get_kms_key.py b/samples/snippets/storage_object_get_kms_key.py index dddfc9151..7604e6eba 100644 --- a/samples/snippets/storage_object_get_kms_key.py +++ b/samples/snippets/storage_object_get_kms_key.py @@ -32,7 +32,7 @@ def object_get_kms_key(bucket_name, blob_name): kms_key = blob.kms_key_name - print("The KMS key of a blob is {}".format(blob.kms_key_name)) + print(f"The KMS key of a blob is {blob.kms_key_name}") return kms_key diff --git a/samples/snippets/storage_print_bucket_acl.py b/samples/snippets/storage_print_bucket_acl.py index 0804f7a9a..55417f1bc 100644 --- a/samples/snippets/storage_print_bucket_acl.py +++ b/samples/snippets/storage_print_bucket_acl.py @@ -27,7 +27,7 @@ def print_bucket_acl(bucket_name): bucket = storage_client.bucket(bucket_name) for entry in bucket.acl: - print("{}: {}".format(entry["role"], entry["entity"])) + print(f"{entry['role']}: {entry['entity']}") # [END storage_print_bucket_acl] diff --git a/samples/snippets/storage_print_file_acl.py b/samples/snippets/storage_print_file_acl.py index f34a5283b..8dfc4e984 100644 --- a/samples/snippets/storage_print_file_acl.py +++ b/samples/snippets/storage_print_file_acl.py @@ -28,7 +28,7 @@ def print_blob_acl(bucket_name, blob_name): blob = bucket.blob(blob_name) for entry in blob.acl: - print("{}: {}".format(entry["role"], entry["entity"])) + print(f"{entry['role']}: {entry['entity']}") # [END storage_print_file_acl] diff --git a/samples/snippets/storage_release_event_based_hold.py b/samples/snippets/storage_release_event_based_hold.py index 8c3c11b6f..1db637cd9 100644 --- a/samples/snippets/storage_release_event_based_hold.py +++ b/samples/snippets/storage_release_event_based_hold.py @@ -33,7 +33,7 @@ def release_event_based_hold(bucket_name, blob_name): blob.event_based_hold = False blob.patch() - print("Event based hold was released for {}".format(blob_name)) + print(f"Event based hold was released for {blob_name}") # [END storage_release_event_based_hold] diff --git a/samples/snippets/storage_remove_bucket_default_owner.py b/samples/snippets/storage_remove_bucket_default_owner.py index beaf6be84..e6f3c495e 100644 --- a/samples/snippets/storage_remove_bucket_default_owner.py +++ b/samples/snippets/storage_remove_bucket_default_owner.py @@ -40,9 +40,7 @@ def remove_bucket_default_owner(bucket_name, user_email): bucket.default_object_acl.save() print( - "Removed user {} from the default acl of bucket {}.".format( - user_email, bucket_name - ) + f"Removed user {user_email} from the default acl of bucket {bucket_name}." ) diff --git a/samples/snippets/storage_remove_bucket_iam_member.py b/samples/snippets/storage_remove_bucket_iam_member.py index ef75a1a15..2efc29e30 100644 --- a/samples/snippets/storage_remove_bucket_iam_member.py +++ b/samples/snippets/storage_remove_bucket_iam_member.py @@ -38,7 +38,7 @@ def remove_bucket_iam_member(bucket_name, role, member): bucket.set_iam_policy(policy) - print("Removed {} with role {} from {}.".format(member, role, bucket_name)) + print(f"Removed {member} with role {role} from {bucket_name}.") # [END storage_remove_bucket_iam_member] diff --git a/samples/snippets/storage_remove_bucket_label.py b/samples/snippets/storage_remove_bucket_label.py index 58bbfef2d..fc4a5b4e7 100644 --- a/samples/snippets/storage_remove_bucket_label.py +++ b/samples/snippets/storage_remove_bucket_label.py @@ -39,7 +39,7 @@ def remove_bucket_label(bucket_name): bucket.labels = labels bucket.patch() - print("Removed labels on {}.".format(bucket.name)) + print(f"Removed labels on {bucket.name}.") pprint.pprint(bucket.labels) diff --git a/samples/snippets/storage_remove_bucket_owner.py b/samples/snippets/storage_remove_bucket_owner.py index f54e7a7cc..561ba9175 100644 --- a/samples/snippets/storage_remove_bucket_owner.py +++ b/samples/snippets/storage_remove_bucket_owner.py @@ -38,7 +38,7 @@ def remove_bucket_owner(bucket_name, user_email): bucket.acl.user(user_email).revoke_owner() bucket.acl.save() - print("Removed user {} from bucket {}.".format(user_email, bucket_name)) + print(f"Removed user {user_email} from bucket {bucket_name}.") # [END storage_remove_bucket_owner] diff --git a/samples/snippets/storage_remove_cors_configuration.py b/samples/snippets/storage_remove_cors_configuration.py index 48ee74338..ad97371f4 100644 --- a/samples/snippets/storage_remove_cors_configuration.py +++ b/samples/snippets/storage_remove_cors_configuration.py @@ -29,7 +29,7 @@ def remove_cors_configuration(bucket_name): bucket.cors = [] bucket.patch() - print("Remove CORS policies for bucket {}.".format(bucket.name)) + print(f"Remove CORS policies for bucket {bucket.name}.") return bucket diff --git a/samples/snippets/storage_remove_file_owner.py b/samples/snippets/storage_remove_file_owner.py index 9db83cce0..315a747ad 100644 --- a/samples/snippets/storage_remove_file_owner.py +++ b/samples/snippets/storage_remove_file_owner.py @@ -39,9 +39,7 @@ def remove_blob_owner(bucket_name, blob_name, user_email): blob.acl.save() print( - "Removed user {} from blob {} in bucket {}.".format( - user_email, blob_name, bucket_name - ) + f"Removed user {user_email} from blob {blob_name} in bucket {bucket_name}." ) diff --git a/samples/snippets/storage_remove_retention_policy.py b/samples/snippets/storage_remove_retention_policy.py index cb8ee548c..9ede8053a 100644 --- a/samples/snippets/storage_remove_retention_policy.py +++ b/samples/snippets/storage_remove_retention_policy.py @@ -37,7 +37,7 @@ def remove_retention_policy(bucket_name): bucket.retention_period = None bucket.patch() - print("Removed bucket {} retention policy".format(bucket.name)) + print(f"Removed bucket {bucket.name} retention policy") # [END storage_remove_retention_policy] diff --git a/samples/snippets/storage_rename_file.py b/samples/snippets/storage_rename_file.py index b47e18621..1125007c6 100644 --- a/samples/snippets/storage_rename_file.py +++ b/samples/snippets/storage_rename_file.py @@ -35,7 +35,7 @@ def rename_blob(bucket_name, blob_name, new_name): new_blob = bucket.rename_blob(blob, new_name) - print("Blob {} has been renamed to {}".format(blob.name, new_blob.name)) + print(f"Blob {blob.name} has been renamed to {new_blob.name}") # [END storage_rename_file] diff --git a/samples/snippets/storage_rotate_encryption_key.py b/samples/snippets/storage_rotate_encryption_key.py index 663ee4796..828b7d5ef 100644 --- a/samples/snippets/storage_rotate_encryption_key.py +++ b/samples/snippets/storage_rotate_encryption_key.py @@ -52,7 +52,7 @@ def rotate_encryption_key( if token is None: break - print("Key rotation complete for Blob {}".format(blob_name)) + print(f"Key rotation complete for Blob {blob_name}") # [END storage_rotate_encryption_key] diff --git a/samples/snippets/storage_set_bucket_public_iam.py b/samples/snippets/storage_set_bucket_public_iam.py index 4b7df89df..0fb33f59c 100644 --- a/samples/snippets/storage_set_bucket_public_iam.py +++ b/samples/snippets/storage_set_bucket_public_iam.py @@ -39,7 +39,7 @@ def set_bucket_public_iam( bucket.set_iam_policy(policy) - print("Bucket {} is now publicly readable".format(bucket.name)) + print(f"Bucket {bucket.name} is now publicly readable") # [END storage_set_bucket_public_iam] diff --git a/samples/snippets/storage_set_event_based_hold.py b/samples/snippets/storage_set_event_based_hold.py index 52a89b88e..e04ed7552 100644 --- a/samples/snippets/storage_set_event_based_hold.py +++ b/samples/snippets/storage_set_event_based_hold.py @@ -32,7 +32,7 @@ def set_event_based_hold(bucket_name, blob_name): blob.event_based_hold = True blob.patch() - print("Event based hold was set for {}".format(blob_name)) + print(f"Event based hold was set for {blob_name}") # [END storage_set_event_based_hold] diff --git a/samples/snippets/storage_set_metadata.py b/samples/snippets/storage_set_metadata.py index 07529ac68..90b6838c0 100644 --- a/samples/snippets/storage_set_metadata.py +++ b/samples/snippets/storage_set_metadata.py @@ -32,7 +32,7 @@ def set_blob_metadata(bucket_name, blob_name): blob.metadata = metadata blob.patch() - print("The metadata for the blob {} is {}".format(blob.name, blob.metadata)) + print(f"The metadata for the blob {blob.name} is {blob.metadata}") # [END storage_set_metadata] diff --git a/samples/snippets/storage_upload_encrypted_file.py b/samples/snippets/storage_upload_encrypted_file.py index e7d02c67b..5f4987238 100644 --- a/samples/snippets/storage_upload_encrypted_file.py +++ b/samples/snippets/storage_upload_encrypted_file.py @@ -51,9 +51,7 @@ def upload_encrypted_blob( blob.upload_from_filename(source_file_name) print( - "File {} uploaded to {}.".format( - source_file_name, destination_blob_name - ) + f"File {source_file_name} uploaded to {destination_blob_name}." ) diff --git a/samples/snippets/storage_upload_file.py b/samples/snippets/storage_upload_file.py index fb02c3632..8e7d98630 100644 --- a/samples/snippets/storage_upload_file.py +++ b/samples/snippets/storage_upload_file.py @@ -36,9 +36,7 @@ def upload_blob(bucket_name, source_file_name, destination_blob_name): blob.upload_from_filename(source_file_name) print( - "File {} uploaded to {}.".format( - source_file_name, destination_blob_name - ) + f"File {source_file_name} uploaded to {destination_blob_name}." ) diff --git a/samples/snippets/storage_upload_from_memory.py b/samples/snippets/storage_upload_from_memory.py index ee8a9828c..eff3d222a 100644 --- a/samples/snippets/storage_upload_from_memory.py +++ b/samples/snippets/storage_upload_from_memory.py @@ -39,9 +39,7 @@ def upload_blob_from_memory(bucket_name, contents, destination_blob_name): blob.upload_from_string(contents) print( - "{} with contents {} uploaded to {}.".format( - destination_blob_name, contents, bucket_name - ) + f"{destination_blob_name} with contents {contents} uploaded to {bucket_name}." ) # [END storage_file_upload_from_memory] diff --git a/samples/snippets/storage_upload_from_stream.py b/samples/snippets/storage_upload_from_stream.py index d43365e08..e2d31a5e3 100644 --- a/samples/snippets/storage_upload_from_stream.py +++ b/samples/snippets/storage_upload_from_stream.py @@ -44,9 +44,7 @@ def upload_blob_from_stream(bucket_name, file_obj, destination_blob_name): blob.upload_from_file(file_obj) print( - "Stream data uploaded to {} in bucket {}.".format( - destination_blob_name, bucket_name - ) + f"Stream data uploaded to {destination_blob_name} in bucket {bucket_name}." ) # [END storage_stream_file_upload] diff --git a/samples/snippets/storage_view_bucket_iam_members.py b/samples/snippets/storage_view_bucket_iam_members.py index 5272f0ddb..184a1361f 100644 --- a/samples/snippets/storage_view_bucket_iam_members.py +++ b/samples/snippets/storage_view_bucket_iam_members.py @@ -30,7 +30,7 @@ def view_bucket_iam_members(bucket_name): policy = bucket.get_iam_policy(requested_policy_version=3) for binding in policy.bindings: - print("Role: {}, Members: {}".format(binding["role"], binding["members"])) + print(f"Role: {binding['role']}, Members: {binding['members']}") # [END storage_view_bucket_iam_members] diff --git a/samples/snippets/uniform_bucket_level_access_test.py b/samples/snippets/uniform_bucket_level_access_test.py index b43fa016f..8b7964038 100644 --- a/samples/snippets/uniform_bucket_level_access_test.py +++ b/samples/snippets/uniform_bucket_level_access_test.py @@ -23,7 +23,7 @@ def test_get_uniform_bucket_level_access(bucket, capsys): ) out, _ = capsys.readouterr() assert ( - "Uniform bucket-level access is disabled for {}.".format(bucket.name) + f"Uniform bucket-level access is disabled for {bucket.name}." in out ) @@ -35,7 +35,7 @@ def test_enable_uniform_bucket_level_access(bucket, capsys): ) out, _ = capsys.readouterr() assert ( - "Uniform bucket-level access was enabled for {}.".format(bucket.name) + f"Uniform bucket-level access was enabled for {bucket.name}." in out ) @@ -47,6 +47,6 @@ def test_disable_uniform_bucket_level_access(bucket, capsys): ) out, _ = capsys.readouterr() assert ( - "Uniform bucket-level access was disabled for {}.".format(bucket.name) + f"Uniform bucket-level access was disabled for {bucket.name}." in out ) diff --git a/tests/conformance/test_conformance.py b/tests/conformance/test_conformance.py index f84131f2f..4d16fc36f 100644 --- a/tests/conformance/test_conformance.py +++ b/tests/conformance/test_conformance.py @@ -42,9 +42,9 @@ """The storage testbench docker image info and commands.""" _DEFAULT_IMAGE_NAME = "gcr.io/cloud-devrel-public-resources/storage-testbench" _DEFAULT_IMAGE_TAG = "latest" -_DOCKER_IMAGE = "{}:{}".format(_DEFAULT_IMAGE_NAME, _DEFAULT_IMAGE_TAG) +_DOCKER_IMAGE = f"{_DEFAULT_IMAGE_NAME}:{_DEFAULT_IMAGE_TAG}" _PULL_CMD = ["docker", "pull", _DOCKER_IMAGE] -_RUN_CMD = ["docker", "run", "--rm", "-d", "-p", "{}:9000".format(_PORT), _DOCKER_IMAGE] +_RUN_CMD = ["docker", "run", "--rm", "-d", "-p", f"{_PORT}:9000", _DOCKER_IMAGE] _CONF_TEST_PROJECT_ID = "my-project-id" _CONF_TEST_SERVICE_ACCOUNT_EMAIL = ( @@ -846,9 +846,7 @@ def _get_retry_test(host, id): instructions, and a boolean status "completed". This can be used to verify if all instructions were used as expected. """ - get_retry_test_uri = "{base}{retry}/{id}".format( - base=host, retry="/retry_test", id=id - ) + get_retry_test_uri = f"{host}/retry_test/{id}" r = requests.get(get_retry_test_uri) return r.json() @@ -892,9 +890,7 @@ def _delete_retry_test(host, id): """ Delete the Retry Test resource by id. """ - get_retry_test_uri = "{base}{retry}/{id}".format( - base=host, retry="/retry_test", id=id - ) + get_retry_test_uri = f"{host}/retry_test/{id}" requests.delete(get_retry_test_uri) @@ -926,7 +922,7 @@ def run_test_case( id = r["id"] except Exception as e: raise Exception( - "Error creating retry test for {}: {}".format(method_name, e) + f"Error creating retry test for {method_name}: {e}" ).with_traceback(e.__traceback__) # Run retry tests on library methods. @@ -943,9 +939,7 @@ def run_test_case( file_data, ) except Exception as e: - logging.exception( - "Caught an exception while running retry instructions\n {}".format(e) - ) + logging.exception(f"Caught an exception while running retry instructions\n {e}") success_results = False else: success_results = True @@ -990,13 +984,11 @@ def run_test_case( method_name = m["name"] method_group = m["group"] if m.get("group", None) else m["name"] if method_group not in method_mapping: - logging.info("No tests for operation {}".format(method_name)) + logging.info(f"No tests for operation {method_name}") continue for lib_func in method_mapping[method_group]: - test_name = "test-S{}-{}-{}-{}".format( - id, method_name, lib_func.__name__, i - ) + test_name = f"test-S{id}-{method_name}-{lib_func.__name__}-{i}" globals()[test_name] = functools.partial( run_test_case, id, m, c, lib_func, _HOST ) diff --git a/tests/system/test_bucket.py b/tests/system/test_bucket.py index de1a04aa9..d8796f5b3 100644 --- a/tests/system/test_bucket.py +++ b/tests/system/test_bucket.py @@ -153,7 +153,7 @@ def test_bucket_get_set_iam_policy( policy = bucket.get_iam_policy(requested_policy_version=3) assert policy == policy_no_version - member = "serviceAccount:{}".format(storage_client.get_service_account_email()) + member = f"serviceAccount:{storage_client.get_service_account_email()}" binding_w_condition = { "role": STORAGE_OBJECT_VIEWER_ROLE, diff --git a/tests/system/test_notification.py b/tests/system/test_notification.py index 59d0dfafd..f52ae3219 100644 --- a/tests/system/test_notification.py +++ b/tests/system/test_notification.py @@ -54,7 +54,7 @@ def topic_name(): @pytest.fixture(scope="session") def topic_path(storage_client, topic_name): - return "projects/{}/topics/{}".format(storage_client.project, topic_name) + return f"projects/{storage_client.project}/topics/{topic_name}" @pytest.fixture(scope="session") @@ -64,7 +64,7 @@ def notification_topic(storage_client, publisher_client, topic_path, no_mtls): binding = policy.bindings.add() binding.role = "roles/pubsub.publisher" binding.members.append( - "serviceAccount:{}".format(storage_client.get_service_account_email()) + f"serviceAccount:{storage_client.get_service_account_email()}" ) publisher_client.set_iam_policy(topic_path, policy) diff --git a/tests/unit/test__http.py b/tests/unit/test__http.py index 890fd1352..9e7bf216b 100644 --- a/tests/unit/test__http.py +++ b/tests/unit/test__http.py @@ -77,7 +77,7 @@ def test_build_api_url_no_extra_query_params(self): conn = self._make_one(object()) uri = conn.build_api_url("/foo") scheme, netloc, path, qs, _ = urlsplit(uri) - self.assertEqual("%s://%s" % (scheme, netloc), conn.API_BASE_URL) + self.assertEqual(f"{scheme}://{netloc}", conn.API_BASE_URL) self.assertEqual(path, "/".join(["", "storage", conn.API_VERSION, "foo"])) parms = dict(parse_qsl(qs)) pretty_print = parms.pop("prettyPrint", "false") @@ -92,7 +92,7 @@ def test_build_api_url_w_custom_endpoint(self): conn = self._make_one(object(), api_endpoint=custom_endpoint) uri = conn.build_api_url("/foo") scheme, netloc, path, qs, _ = urlsplit(uri) - self.assertEqual("%s://%s" % (scheme, netloc), custom_endpoint) + self.assertEqual(f"{scheme}://{netloc}", custom_endpoint) self.assertEqual(path, "/".join(["", "storage", conn.API_VERSION, "foo"])) parms = dict(parse_qsl(qs)) pretty_print = parms.pop("prettyPrint", "false") @@ -106,7 +106,7 @@ def test_build_api_url_w_extra_query_params(self): conn = self._make_one(object()) uri = conn.build_api_url("/foo", {"bar": "baz"}) scheme, netloc, path, qs, _ = urlsplit(uri) - self.assertEqual("%s://%s" % (scheme, netloc), conn.API_BASE_URL) + self.assertEqual(f"{scheme}://{netloc}", conn.API_BASE_URL) self.assertEqual(path, "/".join(["", "storage", conn.API_VERSION, "foo"])) parms = dict(parse_qsl(qs)) self.assertEqual(parms["bar"], "baz") @@ -246,7 +246,7 @@ def test_duplicate_user_agent(self): client_info = ClientInfo(user_agent="test/123") conn = self._make_one(object(), client_info=client_info) - expected_user_agent = "test/123 gcloud-python/{} ".format(__version__) + expected_user_agent = f"test/123 gcloud-python/{__version__} " self.assertEqual(conn._client_info.user_agent, expected_user_agent) client = mock.Mock(_connection=conn, spec=["_connection"]) diff --git a/tests/unit/test__signing.py b/tests/unit/test__signing.py index 48c9a00e1..a7fed514d 100644 --- a/tests/unit/test__signing.py +++ b/tests/unit/test__signing.py @@ -326,7 +326,7 @@ def test_w_query_parameters(self): query_parameters = {"foo": "bar", "baz": "qux"} canonical = self._call_fut(method, resource, query_parameters, None) self.assertEqual(canonical.method, method) - self.assertEqual(canonical.resource, "{}?baz=qux&foo=bar".format(resource)) + self.assertEqual(canonical.resource, f"{resource}?baz=qux&foo=bar") self.assertEqual(canonical.query_parameters, [("baz", "qux"), ("foo", "bar")]) self.assertEqual(canonical.headers, []) @@ -399,7 +399,7 @@ def _generate_helper( for key, value in query_parameters.items() } expected_qp = urlencode(sorted(normalized_qp.items())) - expected_resource = "{}?{}".format(resource, expected_qp) + expected_resource = f"{resource}?{expected_qp}" elements.append(content_md5 or "") elements.append(content_type or "") @@ -568,9 +568,7 @@ def _generate_helper( self.assertEqual(params["X-Goog-Algorithm"], "GOOG4-RSA-SHA256") now_date = now.date().strftime("%Y%m%d") - expected_cred = "{}/{}/auto/storage/goog4_request".format( - signer_email, now_date - ) + expected_cred = f"{signer_email}/{now_date}/auto/storage/goog4_request" self.assertEqual(params["X-Goog-Credential"], expected_cred) now_stamp = now.strftime("%Y%m%dT%H%M%SZ") @@ -859,7 +857,7 @@ def test_conformance_bucket(test_data): resource = "/" _run_conformance_test(resource, test_data, _API_ACCESS_ENDPOINT) else: - resource = "/{}".format(test_data["bucket"]) + resource = f"/{test_data['bucket']}" _run_conformance_test(resource, test_data) @@ -876,14 +874,12 @@ def test_conformance_blob(test_data): # For the VIRTUAL_HOSTED_STYLE else: _API_ACCESS_ENDPOINT = ( - "{scheme}://{bucket_name}.storage.googleapis.com".format( - scheme=test_data["scheme"], bucket_name=test_data["bucket"] - ) + f"{test_data['scheme']}://{test_data['bucket']}.storage.googleapis.com" ) - resource = "/{}".format(test_data["object"]) + resource = f"/{test_data['object']}" _run_conformance_test(resource, test_data, _API_ACCESS_ENDPOINT) else: - resource = "/{}/{}".format(test_data["bucket"], test_data["object"]) + resource = f"/{test_data['bucket']}/{test_data['object']}" _run_conformance_test(resource, test_data) diff --git a/tests/unit/test_acl.py b/tests/unit/test_acl.py index 6083ef1e1..3c5e6515a 100644 --- a/tests/unit/test_acl.py +++ b/tests/unit/test_acl.py @@ -56,7 +56,7 @@ def test___str__w_identifier(self): TYPE = "type" ID = "id" entity = self._make_one(TYPE, ID) - self.assertEqual(str(entity), "%s-%s" % (TYPE, ID)) + self.assertEqual(str(entity), f"{TYPE}-{ID}") def test_grant_simple(self): TYPE = "type" @@ -229,7 +229,7 @@ def test___iter___non_empty_w_roles(self): acl.loaded = True entity = acl.entity(TYPE, ID) entity.grant(ROLE) - self.assertEqual(list(acl), [{"entity": "%s-%s" % (TYPE, ID), "role": ROLE}]) + self.assertEqual(list(acl), [{"entity": f"{TYPE}-{ID}", "role": ROLE}]) def test___iter___non_empty_w_empty_role(self): TYPE = "type" @@ -313,7 +313,7 @@ def test_has_entity_hit_str(self): acl = self._make_one() acl.loaded = True acl.entity(TYPE, ID) - self.assertTrue(acl.has_entity("%s-%s" % (TYPE, ID))) + self.assertTrue(acl.has_entity(f"{TYPE}-{ID}")) def test_has_entity_hit_entity(self): TYPE = "type" @@ -371,7 +371,7 @@ def test_get_entity_hit_str(self): acl = self._make_one() acl.loaded = True acl.entity(TYPE, ID) - self.assertTrue(acl.has_entity("%s-%s" % (TYPE, ID))) + self.assertTrue(acl.has_entity(f"{TYPE}-{ID}")) def test_get_entity_hit_entity(self): TYPE = "type" @@ -422,7 +422,7 @@ def test_add_entity_hit(self): TYPE = "type" ID = "id" - ENTITY_VAL = "%s-%s" % (TYPE, ID) + ENTITY_VAL = f"{TYPE}-{ID}" ROLE = "role" entity = _ACLEntity(TYPE, ID) entity.grant(ROLE) @@ -470,7 +470,7 @@ def test_user(self): entity.grant(ROLE) self.assertEqual(entity.type, "user") self.assertEqual(entity.identifier, ID) - self.assertEqual(list(acl), [{"entity": "user-%s" % ID, "role": ROLE}]) + self.assertEqual(list(acl), [{"entity": f"user-{ID}", "role": ROLE}]) def test_group(self): ID = "id" @@ -481,7 +481,7 @@ def test_group(self): entity.grant(ROLE) self.assertEqual(entity.type, "group") self.assertEqual(entity.identifier, ID) - self.assertEqual(list(acl), [{"entity": "group-%s" % ID, "role": ROLE}]) + self.assertEqual(list(acl), [{"entity": f"group-{ID}", "role": ROLE}]) def test_domain(self): ID = "id" @@ -492,7 +492,7 @@ def test_domain(self): entity.grant(ROLE) self.assertEqual(entity.type, "domain") self.assertEqual(entity.identifier, ID) - self.assertEqual(list(acl), [{"entity": "domain-%s" % ID, "role": ROLE}]) + self.assertEqual(list(acl), [{"entity": f"domain-{ID}", "role": ROLE}]) def test_all(self): ROLE = "role" @@ -1003,8 +1003,8 @@ def test_ctor(self): self.assertEqual(acl.entities, {}) self.assertFalse(acl.loaded) self.assertIs(acl.bucket, bucket) - self.assertEqual(acl.reload_path, "/b/%s/acl" % NAME) - self.assertEqual(acl.save_path, "/b/%s" % NAME) + self.assertEqual(acl.reload_path, f"/b/{NAME}/acl") + self.assertEqual(acl.save_path, f"/b/{NAME}") def test_user_project(self): NAME = "name" @@ -1033,8 +1033,8 @@ def test_ctor(self): self.assertEqual(acl.entities, {}) self.assertFalse(acl.loaded) self.assertIs(acl.bucket, bucket) - self.assertEqual(acl.reload_path, "/b/%s/defaultObjectAcl" % NAME) - self.assertEqual(acl.save_path, "/b/%s" % NAME) + self.assertEqual(acl.reload_path, f"/b/{NAME}/defaultObjectAcl") + self.assertEqual(acl.save_path, f"/b/{NAME}") class Test_ObjectACL(unittest.TestCase): @@ -1056,8 +1056,8 @@ def test_ctor(self): self.assertEqual(acl.entities, {}) self.assertFalse(acl.loaded) self.assertIs(acl.blob, blob) - self.assertEqual(acl.reload_path, "/b/%s/o/%s/acl" % (NAME, BLOB_NAME)) - self.assertEqual(acl.save_path, "/b/%s/o/%s" % (NAME, BLOB_NAME)) + self.assertEqual(acl.reload_path, f"/b/{NAME}/o/{BLOB_NAME}/acl") + self.assertEqual(acl.save_path, f"/b/{NAME}/o/{BLOB_NAME}") def test_user_project(self): NAME = "name" @@ -1081,7 +1081,7 @@ def __init__(self, bucket, blob): @property def path(self): - return "%s/o/%s" % (self.bucket.path, self.blob) + return f"{self.bucket.path}/o/{self.blob}" class _Bucket(object): @@ -1093,4 +1093,4 @@ def __init__(self, name): @property def path(self): - return "/b/%s" % self.name + return f"/b/{self.name}" diff --git a/tests/unit/test_batch.py b/tests/unit/test_batch.py index 8b347fcf8..72b54769f 100644 --- a/tests/unit/test_batch.py +++ b/tests/unit/test_batch.py @@ -280,7 +280,7 @@ def _check_subrequest_no_payload(self, chunk, method, url): self.assertEqual(lines[1], "Content-Type: application/http") self.assertEqual(lines[2], "MIME-Version: 1.0") self.assertEqual(lines[3], "") - self.assertEqual(lines[4], "%s %s HTTP/1.1" % (method, url)) + self.assertEqual(lines[4], f"{method} {url} HTTP/1.1") self.assertEqual(lines[5], "") self.assertEqual(lines[6], "") @@ -294,14 +294,14 @@ def _check_subrequest_payload(self, chunk, method, url, payload): self.assertEqual(lines[1], "Content-Type: application/http") self.assertEqual(lines[2], "MIME-Version: 1.0") self.assertEqual(lines[3], "") - self.assertEqual(lines[4], "%s %s HTTP/1.1" % (method, url)) + self.assertEqual(lines[4], f"{method} {url} HTTP/1.1") if method == "GET": self.assertEqual(len(lines), 7) self.assertEqual(lines[5], "") self.assertEqual(lines[6], "") else: self.assertEqual(len(lines), 9) - self.assertEqual(lines[5], "Content-Length: %d" % len(payload_str)) + self.assertEqual(lines[5], f"Content-Length: {len(payload_str)}") self.assertEqual(lines[6], "Content-Type: application/json") self.assertEqual(lines[7], "") self.assertEqual(json.loads(lines[8]), payload) @@ -352,7 +352,7 @@ def test_finish_nonempty(self): self.assertEqual(response3.headers, {"Content-Length": "0"}) self.assertEqual(response3.status_code, NO_CONTENT) - expected_url = "{}/batch/storage/v1".format(batch.API_BASE_URL) + expected_url = f"{batch.API_BASE_URL}/batch/storage/v1" http.request.assert_called_once_with( method="POST", url=expected_url, @@ -422,7 +422,7 @@ def test_finish_nonempty_with_status_failure(self): self.assertEqual(target1._properties, {"foo": 1, "bar": 2}) self.assertIs(target2._properties, target2_future_before) - expected_url = "{}/batch/storage/v1".format(batch.API_BASE_URL) + expected_url = f"{batch.API_BASE_URL}/batch/storage/v1" http.request.assert_called_once_with( method="POST", url=expected_url, diff --git a/tests/unit/test_blob.py b/tests/unit/test_blob.py index 8c86c002e..cea384846 100644 --- a/tests/unit/test_blob.py +++ b/tests/unit/test_blob.py @@ -140,7 +140,7 @@ def _set_properties_helper(self, kms_key_name=None): NOW = now.strftime(_RFC3339_MICROS) BLOB_NAME = "blob-name" GENERATION = 12345 - BLOB_ID = "name/{}/{}".format(BLOB_NAME, GENERATION) + BLOB_ID = f"name/{BLOB_NAME}/{GENERATION}" SELF_LINK = "http://example.com/self/" METAGENERATION = 23456 SIZE = 12345 @@ -321,7 +321,7 @@ def test_path_normal(self): BLOB_NAME = "blob-name" bucket = _Bucket() blob = self._make_one(BLOB_NAME, bucket=bucket) - self.assertEqual(blob.path, "/b/name/o/%s" % BLOB_NAME) + self.assertEqual(blob.path, f"/b/name/o/{BLOB_NAME}") def test_path_w_slash_in_name(self): BLOB_NAME = "parent/child" @@ -402,7 +402,7 @@ def test_public_url(self): bucket = _Bucket() blob = self._make_one(BLOB_NAME, bucket=bucket) self.assertEqual( - blob.public_url, "https://storage.googleapis.com/name/%s" % BLOB_NAME + blob.public_url, f"https://storage.googleapis.com/name/{BLOB_NAME}" ) def test_public_url_w_slash_in_name(self): @@ -486,9 +486,7 @@ def _generate_signed_url_helper( else: effective_version = version - to_patch = "google.cloud.storage.blob.generate_signed_url_{}".format( - effective_version - ) + to_patch = f"google.cloud.storage.blob.generate_signed_url_{effective_version}" with mock.patch(to_patch) as signer: signed_uri = blob.generate_signed_url( @@ -525,10 +523,10 @@ def _generate_signed_url_helper( ) else: expected_api_access_endpoint = api_access_endpoint - expected_resource = "/{}/{}".format(bucket.name, quoted_name) + expected_resource = f"/{bucket.name}/{quoted_name}" if virtual_hosted_style or bucket_bound_hostname: - expected_resource = "/{}".format(quoted_name) + expected_resource = f"/{quoted_name}" if encryption_key is not None: expected_headers = headers or {} @@ -946,7 +944,7 @@ def test__get_download_url_with_generation_match(self): ) self.assertEqual( download_url, - "{}?ifGenerationMatch={}".format(MEDIA_LINK, GENERATION_NUMBER), + f"{MEDIA_LINK}?ifGenerationMatch={GENERATION_NUMBER}", ) def test__get_download_url_with_media_link_w_user_project(self): @@ -962,9 +960,7 @@ def test__get_download_url_with_media_link_w_user_project(self): client._connection.API_BASE_URL = "https://storage.googleapis.com" download_url = blob._get_download_url(client) - self.assertEqual( - download_url, "{}?userProject={}".format(media_link, user_project) - ) + self.assertEqual(download_url, f"{media_link}?userProject={user_project}") def test__get_download_url_on_the_fly(self): blob_name = "bzzz-fly.txt" @@ -1212,7 +1208,7 @@ def _do_download_helper_wo_chunks( start=1, end=3, raw_download=raw_download, - **extra_kwargs + **extra_kwargs, ) else: blob._do_download( @@ -1221,7 +1217,7 @@ def _do_download_helper_wo_chunks( download_url, headers, raw_download=raw_download, - **extra_kwargs + **extra_kwargs, ) if w_range: @@ -1350,7 +1346,7 @@ def side_effect(*args, **kwargs): end=3, raw_download=raw_download, checksum=checksum, - **timeout_kwarg + **timeout_kwarg, ) else: blob._do_download( @@ -1360,7 +1356,7 @@ def side_effect(*args, **kwargs): headers, raw_download=raw_download, checksum=checksum, - **timeout_kwarg + **timeout_kwarg, ) if w_range: @@ -1616,7 +1612,7 @@ def _download_to_filename_helper( temp.name, raw_download=raw_download, timeout=timeout, - **extra_kwargs + **extra_kwargs, ) if updated is None: @@ -1906,7 +1902,7 @@ def _download_as_text_helper( no_charset=False, expected_value="DEADBEEF", payload=None, - **extra_kwargs + **extra_kwargs, ): if payload is None: if encoding is not None: @@ -1920,7 +1916,7 @@ def _download_as_text_helper( properties = {} if charset is not None: - properties["contentType"] = "text/plain; charset={}".format(charset) + properties["contentType"] = f"text/plain; charset={charset}" elif no_charset: properties = {"contentType": "text/plain"} @@ -2337,7 +2333,7 @@ def _do_multipart_success( if_metageneration_match, if_metageneration_not_match, retry=retry, - **timeout_kwarg + **timeout_kwarg, ) # Clean up the get_api_base_url_for_mtls mock. @@ -2610,7 +2606,7 @@ def _initiate_resumable_helper( if_metageneration_match=if_metageneration_match, if_metageneration_not_match=if_metageneration_not_match, retry=retry, - **timeout_kwarg + **timeout_kwarg, ) # Clean up the get_api_base_url_for_mtls mock. @@ -2815,7 +2811,7 @@ def _make_resumable_transport( fake_response2 = self._mock_requests_response( resumable_media.PERMANENT_REDIRECT, headers2 ) - json_body = '{{"size": "{:d}"}}'.format(total_bytes) + json_body = f'{{"size": "{total_bytes:d}"}}' if data_corruption: fake_response3 = resumable_media.DataCorruption(None) else: @@ -2847,7 +2843,7 @@ def _do_resumable_upload_call0( + "/o?uploadType=resumable" ) if predefined_acl is not None: - upload_url += "&predefinedAcl={}".format(predefined_acl) + upload_url += f"&predefinedAcl={predefined_acl}" expected_headers = _get_default_headers( client._connection.user_agent, x_upload_content_type=content_type ) @@ -2875,9 +2871,9 @@ def _do_resumable_upload_call1( ): # Second mock transport.request() does sends first chunk. if size is None: - content_range = "bytes 0-{:d}/*".format(blob.chunk_size - 1) + content_range = f"bytes 0-{blob.chunk_size - 1:}/*" else: - content_range = "bytes 0-{:d}/{:d}".format(blob.chunk_size - 1, size) + content_range = f"bytes 0-{blob.chunk_size - 1}/{size}" expected_headers = { **_get_default_headers( @@ -2911,9 +2907,7 @@ def _do_resumable_upload_call2( timeout=None, ): # Third mock transport.request() does sends last chunk. - content_range = "bytes {:d}-{:d}/{:d}".format( - blob.chunk_size, total_bytes - 1, total_bytes - ) + content_range = f"bytes {blob.chunk_size:d}-{total_bytes - 1:d}/{total_bytes:d}" expected_headers = { **_get_default_headers( client._connection.user_agent, x_upload_content_type=content_type @@ -2965,7 +2959,7 @@ def _do_resumable_helper( } headers2 = { **_get_default_headers(USER_AGENT, content_type), - "range": "bytes=0-{:d}".format(CHUNK_SIZE - 1), + "range": f"bytes=0-{CHUNK_SIZE - 1:d}", } headers3 = _get_default_headers(USER_AGENT, content_type) transport, responses = self._make_resumable_transport( @@ -3010,7 +3004,7 @@ def _do_resumable_helper( if_metageneration_match, if_metageneration_not_match, retry=retry, - **timeout_kwarg + **timeout_kwarg, ) # Check the returned values. @@ -3148,7 +3142,7 @@ def _do_upload_helper( if_metageneration_match, if_metageneration_not_match, retry=retry, - **timeout_kwarg + **timeout_kwarg, ) if retry is DEFAULT_RETRY_IF_GENERATION_SPECIFIED: @@ -3539,7 +3533,7 @@ def _upload_from_string_helper(self, data, **kwargs): "text/plain", len(payload), kwargs.get("timeout", self._get_default_timeout()), - **extra_kwargs + **extra_kwargs, ) self.assertIsInstance(stream, io.BytesIO) self.assertEqual(stream.getvalue(), payload) @@ -3622,7 +3616,7 @@ def _create_resumable_upload_session_helper( if_metageneration_match=if_metageneration_match, if_metageneration_not_match=if_metageneration_not_match, retry=retry, - **timeout_kwarg + **timeout_kwarg, ) # Check the returned value and (lack of) side-effect. @@ -3721,7 +3715,7 @@ def test_get_iam_policy_defaults(self): from google.api_core.iam import Policy blob_name = "blob-name" - path = "/b/name/o/%s" % (blob_name,) + path = f"/b/name/o/{blob_name}" etag = "DEADBEEF" version = 1 owner1 = "user:phred@example.com" @@ -3756,7 +3750,7 @@ def test_get_iam_policy_defaults(self): self.assertEqual(policy.version, api_response["version"]) self.assertEqual(dict(policy), expected_policy) - expected_path = "%s/iam" % (path,) + expected_path = f"{path}/iam" expected_query_params = {} client._get_resource.assert_called_once_with( expected_path, @@ -3772,7 +3766,7 @@ def test_get_iam_policy_w_user_project_w_timeout(self): blob_name = "blob-name" user_project = "user-project-123" timeout = 42 - path = "/b/name/o/%s" % (blob_name,) + path = f"/b/name/o/{blob_name}" etag = "DEADBEEF" version = 1 api_response = { @@ -3794,7 +3788,7 @@ def test_get_iam_policy_w_user_project_w_timeout(self): self.assertEqual(policy.version, api_response["version"]) self.assertEqual(dict(policy), expected_policy) - expected_path = "%s/iam" % (path,) + expected_path = f"{path}/iam" expected_query_params = {"userProject": user_project} client._get_resource.assert_called_once_with( expected_path, @@ -3808,7 +3802,7 @@ def test_get_iam_policy_w_requested_policy_version(self): from google.cloud.storage.iam import STORAGE_OWNER_ROLE blob_name = "blob-name" - path = "/b/name/o/%s" % (blob_name,) + path = f"/b/name/o/{blob_name}" etag = "DEADBEEF" version = 3 owner1 = "user:phred@example.com" @@ -3828,7 +3822,7 @@ def test_get_iam_policy_w_requested_policy_version(self): self.assertEqual(policy.version, version) - expected_path = "%s/iam" % (path,) + expected_path = f"{path}/iam" expected_query_params = {"optionsRequestedPolicyVersion": version} client._get_resource.assert_called_once_with( expected_path, @@ -3846,7 +3840,7 @@ def test_set_iam_policy(self): from google.api_core.iam import Policy blob_name = "blob-name" - path = "/b/name/o/%s" % (blob_name,) + path = f"/b/name/o/{blob_name}" etag = "DEADBEEF" version = 1 owner1 = "user:phred@example.com" @@ -3876,7 +3870,7 @@ def test_set_iam_policy(self): self.assertEqual(returned.version, version) self.assertEqual(dict(returned), dict(policy)) - expected_path = "%s/iam" % (path,) + expected_path = f"{path}/iam" expected_data = { "resourceId": path, "bindings": mock.ANY, @@ -3904,7 +3898,7 @@ def test_set_iam_policy_w_user_project_w_explicit_client_w_timeout_retry(self): blob_name = "blob-name" user_project = "user-project-123" - path = "/b/name/o/%s" % (blob_name,) + path = f"/b/name/o/{blob_name}" etag = "DEADBEEF" version = 1 bindings = [] @@ -3929,7 +3923,7 @@ def test_set_iam_policy_w_user_project_w_explicit_client_w_timeout_retry(self): self.assertEqual(returned.version, version) self.assertEqual(dict(returned), dict(policy)) - expected_path = "%s/iam" % (path,) + expected_path = f"{path}/iam" expected_data = { # bindings omitted "resourceId": path, } @@ -3965,7 +3959,7 @@ def test_test_iam_permissions_defaults(self): self.assertEqual(found, expected) - expected_path = "/b/name/o/%s/iam/testPermissions" % (blob_name,) + expected_path = f"/b/name/o/{blob_name}/iam/testPermissions" expected_query_params = {"permissions": permissions} client._get_resource.assert_called_once_with( expected_path, @@ -4000,7 +3994,7 @@ def test_test_iam_permissions_w_user_project_w_timeout_w_retry(self): self.assertEqual(found, expected) - expected_path = "/b/name/o/%s/iam/testPermissions" % (blob_name,) + expected_path = f"/b/name/o/{blob_name}/iam/testPermissions" expected_query_params = { "permissions": permissions, "userProject": user_project, @@ -4190,7 +4184,7 @@ def test_compose_wo_content_type_set(self): self.assertIsNone(destination.content_type) - expected_path = "/b/name/o/%s/compose" % destination_name + expected_path = f"/b/name/o/{destination_name}/compose" expected_data = { "sourceObjects": [ {"name": source_1.name, "generation": source_1.generation}, @@ -4227,7 +4221,7 @@ def test_compose_minimal_w_user_project_w_timeout(self): self.assertEqual(destination.etag, "DEADBEEF") - expected_path = "/b/name/o/%s/compose" % destination_name + expected_path = f"/b/name/o/{destination_name}/compose" expected_data = { "sourceObjects": [ {"name": source_1.name, "generation": source_1.generation}, @@ -4265,7 +4259,7 @@ def test_compose_w_additional_property_changes_w_retry(self): self.assertEqual(destination.etag, "DEADBEEF") - expected_path = "/b/name/o/%s/compose" % destination_name + expected_path = f"/b/name/o/{destination_name}/compose" expected_data = { "sourceObjects": [ {"name": source_1.name, "generation": source_1.generation}, @@ -4306,7 +4300,7 @@ def test_compose_w_source_generation_match(self): if_source_generation_match=source_generation_numbers, ) - expected_path = "/b/name/o/%s/compose" % destination_name + expected_path = f"/b/name/o/{destination_name}/compose" expected_data = { "sourceObjects": [ { @@ -4374,7 +4368,7 @@ def test_compose_w_source_generation_match_nones(self): if_source_generation_match=source_generation_numbers, ) - expected_path = "/b/name/o/%s/compose" % destination_name + expected_path = f"/b/name/o/{destination_name}/compose" expected_data = { "sourceObjects": [ { @@ -4416,7 +4410,7 @@ def test_compose_w_generation_match(self): if_generation_match=generation_number, ) - expected_path = "/b/name/o/%s/compose" % destination_name + expected_path = f"/b/name/o/{destination_name}/compose" expected_data = { "sourceObjects": [ {"name": source_1.name, "generation": source_1.generation}, @@ -4456,7 +4450,7 @@ def test_compose_w_if_generation_match_list_w_warning(self, mock_warn): if_generation_match=generation_numbers, ) - expected_path = "/b/name/o/%s/compose" % destination_name + expected_path = f"/b/name/o/{destination_name}/compose" expected_data = { "sourceObjects": [ { @@ -4542,7 +4536,7 @@ def test_compose_w_if_metageneration_match_list_w_warning(self, mock_warn): if_metageneration_match=metageneration_number, ) - expected_path = "/b/name/o/%s/compose" % destination_name + expected_path = f"/b/name/o/{destination_name}/compose" expected_data = { "sourceObjects": [ {"name": source_1_name, "generation": None}, @@ -4584,7 +4578,7 @@ def test_compose_w_metageneration_match(self): if_metageneration_match=metageneration_number, ) - expected_path = "/b/name/o/%s/compose" % destination_name + expected_path = f"/b/name/o/{destination_name}/compose" expected_data = { "sourceObjects": [ {"name": source_1.name, "generation": source_1.generation}, @@ -4830,7 +4824,7 @@ def test_rewrite_same_name_no_old_key_new_key_done_w_user_project(self): self.assertEqual(rewritten, bytes_rewritten) self.assertEqual(size, object_size) - expected_path = "/b/name/o/%s/rewriteTo/b/name/o/%s" % (blob_name, blob_name) + expected_path = f"/b/name/o/{blob_name}/rewriteTo/b/name/o/{blob_name}" expected_query_params = {"userProject": user_project} expected_data = {} expected_headers = { @@ -4878,7 +4872,7 @@ def test_rewrite_same_name_no_key_new_key_w_token(self): self.assertEqual(rewritten, bytes_rewritten) self.assertEqual(size, object_size) - expected_path = "/b/name/o/%s/rewriteTo/b/name/o/%s" % (blob_name, blob_name) + expected_path = f"/b/name/o/{blob_name}/rewriteTo/b/name/o/{blob_name}" expected_data = {} expected_query_params = {"rewriteToken": previous_token} expected_headers = { @@ -4930,7 +4924,7 @@ def test_rewrite_same_name_w_old_key_new_kms_key(self): self.assertEqual(rewritten, bytes_rewritten) self.assertEqual(size, object_size) - expected_path = "/b/name/o/%s/rewriteTo/b/name/o/%s" % (blob_name, blob_name) + expected_path = f"/b/name/o/{blob_name}/rewriteTo/b/name/o/{blob_name}" expected_data = {"kmsKeyName": dest_kms_resource} expected_query_params = {"destinationKmsKeyName": dest_kms_resource} expected_headers = { @@ -5799,7 +5793,7 @@ def _helper(self, message, code=http.client.BAD_REQUEST, reason=None, args=()): def test_default(self): message = "Failure" exc_info = self._helper(message) - expected = "GET http://example.com/: {}".format(message) + expected = f"GET http://example.com/: {message}" self.assertEqual(exc_info.exception.message, expected) self.assertEqual(exc_info.exception.errors, []) @@ -5831,18 +5825,14 @@ def test_w_empty_list(self): def test_wo_existing_qs(self): BASE_URL = "https://test.example.com/base" NV_LIST = [("one", "One"), ("two", "Two")] - expected = "&".join(["{}={}".format(name, value) for name, value in NV_LIST]) - self.assertEqual( - self._call_fut(BASE_URL, NV_LIST), "{}?{}".format(BASE_URL, expected) - ) + expected = "&".join([f"{name}={value}" for name, value in NV_LIST]) + self.assertEqual(self._call_fut(BASE_URL, NV_LIST), f"{BASE_URL}?{expected}") def test_w_existing_qs(self): BASE_URL = "https://test.example.com/base?one=Three" NV_LIST = [("one", "One"), ("two", "Two")] - expected = "&".join(["{}={}".format(name, value) for name, value in NV_LIST]) - self.assertEqual( - self._call_fut(BASE_URL, NV_LIST), "{}&{}".format(BASE_URL, expected) - ) + expected = "&".join([f"{name}={value}" for name, value in NV_LIST]) + self.assertEqual(self._call_fut(BASE_URL, NV_LIST), f"{BASE_URL}&{expected}") class _Connection(object): diff --git a/tests/unit/test_bucket.py b/tests/unit/test_bucket.py index eb402de9e..f253db3e1 100644 --- a/tests/unit/test_bucket.py +++ b/tests/unit/test_bucket.py @@ -902,7 +902,7 @@ def test_path_no_name(self): def test_path_w_name(self): NAME = "name" bucket = self._make_one(name=NAME) - self.assertEqual(bucket.path, "/b/%s" % NAME) + self.assertEqual(bucket.path, f"/b/{NAME}") def test_get_blob_miss_w_defaults(self): from google.cloud.exceptions import NotFound @@ -918,7 +918,7 @@ def test_get_blob_miss_w_defaults(self): self.assertIsNone(result) - expected_path = "/b/%s/o/%s" % (name, blob_name) + expected_path = f"/b/{name}/o/{blob_name}" expected_query_params = {"projection": "noAcl"} expected_headers = {} client._get_resource.assert_called_once_with( @@ -952,7 +952,7 @@ def test_get_blob_hit_w_user_project(self): self.assertIs(blob.bucket, bucket) self.assertEqual(blob.name, blob_name) - expected_path = "/b/%s/o/%s" % (name, blob_name) + expected_path = f"/b/{name}/o/{blob_name}" expected_query_params = { "userProject": user_project, "projection": "noAcl", @@ -986,7 +986,7 @@ def test_get_blob_hit_w_generation_w_timeout(self): self.assertEqual(blob.name, blob_name) self.assertEqual(blob.generation, generation) - expected_path = "/b/%s/o/%s" % (name, blob_name) + expected_path = f"/b/{name}/o/{blob_name}" expected_query_params = { "generation": generation, "projection": "noAcl", @@ -1020,7 +1020,7 @@ def test_get_blob_w_etag_match_w_retry(self): self.assertEqual(blob.name, blob_name) self.assertEqual(blob.etag, etag) - expected_path = "/b/%s/o/%s" % (name, blob_name) + expected_path = f"/b/{name}/o/{blob_name}" expected_query_params = { "projection": "noAcl", } @@ -1055,7 +1055,7 @@ def test_get_blob_w_generation_match_w_retry(self): self.assertEqual(blob.name, blob_name) self.assertEqual(blob.generation, generation) - expected_path = "/b/%s/o/%s" % (name, blob_name) + expected_path = f"/b/{name}/o/{blob_name}" expected_query_params = { "ifGenerationMatch": generation, "projection": "noAcl", @@ -1093,7 +1093,7 @@ def test_get_blob_hit_with_kwargs_w_explicit_client(self): self.assertEqual(blob.chunk_size, chunk_size) self.assertEqual(blob._encryption_key, key) - expected_path = "/b/%s/o/%s" % (name, blob_name) + expected_path = f"/b/{name}/o/{blob_name}" expected_query_params = { "projection": "noAcl", } @@ -1218,7 +1218,7 @@ def test_list_notifications_w_defaults(self): self.assertIs(iterator, client._list_resource.return_value) self.assertIs(iterator.bucket, bucket) - expected_path = "/b/{}/notificationConfigs".format(bucket_name) + expected_path = f"/b/{bucket_name}/notificationConfigs" expected_item_to_value = _item_to_notification client._list_resource.assert_called_once_with( expected_path, @@ -1246,7 +1246,7 @@ def test_list_notifications_w_explicit(self): self.assertIs(iterator, other_client._list_resource.return_value) self.assertIs(iterator.bucket, bucket) - expected_path = "/b/{}/notificationConfigs".format(bucket_name) + expected_path = f"/b/{bucket_name}/notificationConfigs" expected_item_to_value = _item_to_notification other_client._list_resource.assert_called_once_with( expected_path, @@ -1270,7 +1270,7 @@ def test_get_notification_miss_w_defaults(self): with self.assertRaises(NotFound): bucket.get_notification(notification_id=notification_id) - expected_path = "/b/{}/notificationConfigs/{}".format(name, notification_id) + expected_path = f"/b/{name}/notificationConfigs/{notification_id}" expected_query_params = {} client._get_resource.assert_called_once_with( expected_path, @@ -1319,7 +1319,7 @@ def test_get_notification_hit_w_explicit_w_user_project(self): self.assertIsNone(notification.blob_name_prefix) self.assertEqual(notification.payload_format, JSON_API_V1_PAYLOAD_FORMAT) - expected_path = "/b/{}/notificationConfigs/{}".format(name, notification_id) + expected_path = f"/b/{name}/notificationConfigs/{notification_id}" expected_query_params = {"userProject": user_project} client._get_resource.assert_called_once_with( expected_path, @@ -1519,7 +1519,7 @@ def test_delete_blob_miss_w_defaults(self): with self.assertRaises(NotFound): bucket.delete_blob(blob_name) - expected_path = "/b/%s/o/%s" % (name, blob_name) + expected_path = f"/b/{name}/o/{blob_name}" expected_query_params = {} client._delete_resource.assert_called_once_with( expected_path, @@ -1542,7 +1542,7 @@ def test_delete_blob_hit_w_user_project_w_timeout(self): self.assertIsNone(result) - expected_path = "/b/%s/o/%s" % (name, blob_name) + expected_path = f"/b/{name}/o/{blob_name}" expected_query_params = {"userProject": user_project} client._delete_resource.assert_called_once_with( expected_path, @@ -1565,7 +1565,7 @@ def test_delete_blob_hit_w_generation_w_retry(self): self.assertIsNone(result) - expected_path = "/b/%s/o/%s" % (name, blob_name) + expected_path = f"/b/{name}/o/{blob_name}" expected_query_params = {"generation": generation} client._delete_resource.assert_called_once_with( expected_path, @@ -1592,7 +1592,7 @@ def test_delete_blob_hit_w_generation_match(self): self.assertIsNone(result) - expected_path = "/b/%s/o/%s" % (name, blob_name) + expected_path = f"/b/{name}/o/{blob_name}" expected_query_params = { "ifGenerationMatch": generation, "ifMetagenerationMatch": metageneration, @@ -1811,7 +1811,7 @@ def test_reload_w_etag_match(self): bucket.reload(if_etag_match=etag) - expected_path = "/b/%s" % (name,) + expected_path = f"/b/{name}" expected_query_params = { "projection": "noAcl", } @@ -1837,7 +1837,7 @@ def test_reload_w_metageneration_match(self): bucket.reload(if_metageneration_match=metageneration_number) - expected_path = "/b/%s" % (name,) + expected_path = f"/b/{name}" expected_query_params = { "projection": "noAcl", "ifMetagenerationMatch": metageneration_number, @@ -1899,7 +1899,7 @@ def _make_blob(bucket_name, blob_name): blob = mock.create_autospec(Blob) blob.name = blob_name - blob.path = "/b/{}/o/{}".format(bucket_name, blob_name) + blob.path = f"/b/{bucket_name}/o/{blob_name}" return blob def test_copy_blobs_wo_name(self): @@ -2048,7 +2048,7 @@ def test_copy_blob_w_preserve_acl_false_w_explicit_client(self): _target_object=new_blob, ) - expected_patch_path = "/b/{}/o/{}".format(dest_name, new_name) + expected_patch_path = f"/b/{dest_name}/o/{new_name}" expected_patch_data = {"acl": []} expected_patch_query_params = {"projection": "full"} client._patch_resource.assert_called_once_with( @@ -2960,7 +2960,7 @@ def test_get_iam_policy_defaults(self): from google.api_core.iam import Policy bucket_name = "name" - path = "/b/%s" % (bucket_name,) + path = f"/b/{bucket_name}" etag = "DEADBEEF" version = 1 owner1 = "user:phred@example.com" @@ -2994,7 +2994,7 @@ def test_get_iam_policy_defaults(self): self.assertEqual(policy.version, api_response["version"]) self.assertEqual(dict(policy), expected_policy) - expected_path = "/b/%s/iam" % (bucket_name,) + expected_path = f"/b/{bucket_name}/iam" expected_query_params = {} client._get_resource.assert_called_once_with( expected_path, @@ -3010,7 +3010,7 @@ def test_get_iam_policy_w_user_project_w_timeout(self): bucket_name = "name" timeout = 42 user_project = "user-project-123" - path = "/b/%s" % (bucket_name,) + path = f"/b/{bucket_name}" etag = "DEADBEEF" version = 1 api_response = { @@ -3033,7 +3033,7 @@ def test_get_iam_policy_w_user_project_w_timeout(self): self.assertEqual(policy.version, api_response["version"]) self.assertEqual(dict(policy), expected_policy) - expected_path = "/b/%s/iam" % (bucket_name,) + expected_path = f"/b/{bucket_name}/iam" expected_query_params = {"userProject": user_project} client._get_resource.assert_called_once_with( expected_path, @@ -3047,7 +3047,7 @@ def test_get_iam_policy_w_requested_policy_version_w_retry(self): from google.cloud.storage.iam import STORAGE_OWNER_ROLE bucket_name = "name" - path = "/b/%s" % (bucket_name,) + path = f"/b/{bucket_name}" etag = "DEADBEEF" version = 3 owner1 = "user:phred@example.com" @@ -3067,7 +3067,7 @@ def test_get_iam_policy_w_requested_policy_version_w_retry(self): self.assertEqual(policy.version, version) - expected_path = "/b/%s/iam" % (bucket_name,) + expected_path = f"/b/{bucket_name}/iam" expected_query_params = {"optionsRequestedPolicyVersion": version} client._get_resource.assert_called_once_with( expected_path, @@ -3113,7 +3113,7 @@ def test_set_iam_policy_w_defaults(self): self.assertEqual(returned.version, version) self.assertEqual(dict(returned), dict(policy)) - expected_path = "%s/iam" % (bucket.path,) + expected_path = f"{bucket.path}/iam" expected_data = { "resourceId": bucket.path, "bindings": mock.ANY, @@ -3177,7 +3177,7 @@ def test_set_iam_policy_w_user_project_w_expl_client_w_timeout_retry(self): self.assertEqual(returned.version, version) self.assertEqual(dict(returned), dict(policy)) - expected_path = "%s/iam" % (bucket.path,) + expected_path = f"{bucket.path}/iam" expected_data = { "resourceId": bucket.path, "bindings": mock.ANY, @@ -3221,7 +3221,7 @@ def test_test_iam_permissions_defaults(self): self.assertEqual(found, expected) - expected_path = "/b/%s/iam/testPermissions" % (name,) + expected_path = f"/b/{name}/iam/testPermissions" expected_query_params = {} expected_query_params = {"permissions": permissions} client._get_resource.assert_called_once_with( @@ -3256,7 +3256,7 @@ def test_test_iam_permissions_w_user_project_w_timeout_w_retry(self): self.assertEqual(found, expected) - expected_path = "/b/%s/iam/testPermissions" % (name,) + expected_path = f"/b/{name}/iam/testPermissions" expected_query_params = { "permissions": permissions, "userProject": user_project, @@ -3369,7 +3369,7 @@ def _make_public_w_future_helper(self, default_object_acl_loaded=True): ) if not default_object_acl_loaded: - expected_path = "/b/%s/defaultObjectAcl" % (name,) + expected_path = f"/b/{name}/defaultObjectAcl" expected_query_params = {} client._get_resource.assert_called_once_with( expected_path, @@ -3581,7 +3581,7 @@ def _make_private_w_future_helper(self, default_object_acl_loaded=True): ) if not default_object_acl_loaded: - expected_path = "/b/%s/defaultObjectAcl" % (name,) + expected_path = f"/b/{name}/defaultObjectAcl" expected_query_params = {} client._get_resource.assert_called_once_with( expected_path, @@ -3724,9 +3724,7 @@ def _generate_upload_policy_helper(self, **kwargs): break else: # pragma: NO COVER self.fail( - "Condition {} not found in {}".format( - expected_condition, policy_conditions - ) + f"Condition {expected_condition} not found in {policy_conditions}" ) return policy_fields, policy @@ -3831,7 +3829,7 @@ def test_lock_retention_policy_ok_w_timeout_w_retry(self): bucket.lock_retention_policy(timeout=timeout, retry=retry) - expected_path = "/b/{}/lockRetentionPolicy".format(name) + expected_path = f"/b/{name}/lockRetentionPolicy" expected_data = None expected_query_params = {"ifMetagenerationMatch": metageneration} client._post_resource.assert_called_once_with( @@ -3869,7 +3867,7 @@ def test_lock_retention_policy_w_user_project(self): bucket.lock_retention_policy() - expected_path = "/b/{}/lockRetentionPolicy".format(name) + expected_path = f"/b/{name}/lockRetentionPolicy" expected_data = None expected_query_params = { "ifMetagenerationMatch": metageneration, @@ -3964,7 +3962,7 @@ def _generate_signed_url_helper( ) else: expected_api_access_endpoint = api_access_endpoint - expected_resource = "/{}".format(parse.quote(bucket_name)) + expected_resource = f"/{parse.quote(bucket_name)}" if virtual_hosted_style or bucket_bound_hostname: expected_resource = "/" diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index 6a97d8d41..07d1b0655 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -371,7 +371,7 @@ def test_get_service_account_email_wo_project(self): ) _, kwargs = http.request.call_args scheme, netloc, path, qs, _ = urllib.parse.urlsplit(kwargs.get("url")) - self.assertEqual("%s://%s" % (scheme, netloc), client._connection.API_BASE_URL) + self.assertEqual(f"{scheme}://{netloc}", client._connection.API_BASE_URL) self.assertEqual( path, "/".join( @@ -409,7 +409,7 @@ def test_get_service_account_email_w_project(self): ) _, kwargs = http.request.call_args scheme, netloc, path, qs, _ = urllib.parse.urlsplit(kwargs.get("url")) - self.assertEqual("%s://%s" % (scheme, netloc), client._connection.API_BASE_URL) + self.assertEqual(f"{scheme}://{netloc}", client._connection.API_BASE_URL) self.assertEqual( path, "/".join( @@ -899,7 +899,7 @@ def test_get_bucket_miss_w_string_w_defaults(self): with self.assertRaises(NotFound): client.get_bucket(bucket_name) - expected_path = "/b/%s" % (bucket_name,) + expected_path = f"/b/{bucket_name}" expected_query_params = {"projection": "noAcl"} expected_headers = {} client._get_resource.assert_called_once_with( @@ -931,7 +931,7 @@ def test_get_bucket_hit_w_string_w_timeout(self): self.assertIsInstance(bucket, Bucket) self.assertEqual(bucket.name, bucket_name) - expected_path = "/b/%s" % (bucket_name,) + expected_path = f"/b/{bucket_name}" expected_query_params = {"projection": "noAcl"} expected_headers = {} client._get_resource.assert_called_once_with( @@ -961,7 +961,7 @@ def test_get_bucket_hit_w_string_w_metageneration_match(self): self.assertIsInstance(bucket, Bucket) self.assertEqual(bucket.name, bucket_name) - expected_path = "/b/%s" % (bucket_name,) + expected_path = f"/b/{bucket_name}" expected_query_params = { "projection": "noAcl", "ifMetagenerationMatch": metageneration_number, @@ -991,7 +991,7 @@ def test_get_bucket_miss_w_object_w_retry(self): with self.assertRaises(NotFound): client.get_bucket(bucket_obj, retry=retry) - expected_path = "/b/%s" % (bucket_name,) + expected_path = f"/b/{bucket_name}" expected_query_params = {"projection": "noAcl"} expected_headers = {} client._get_resource.assert_called_once_with( @@ -1023,7 +1023,7 @@ def test_get_bucket_hit_w_object_defaults(self): self.assertIsInstance(bucket, Bucket) self.assertEqual(bucket.name, bucket_name) - expected_path = "/b/%s" % (bucket_name,) + expected_path = f"/b/{bucket_name}" expected_query_params = {"projection": "noAcl"} expected_headers = {} client._get_resource.assert_called_once_with( @@ -1051,7 +1051,7 @@ def test_get_bucket_hit_w_object_w_retry_none(self): self.assertIsInstance(bucket, Bucket) self.assertEqual(bucket.name, bucket_name) - expected_path = "/b/%s" % (bucket_name,) + expected_path = f"/b/{bucket_name}" expected_query_params = {"projection": "noAcl"} expected_headers = {} client._get_resource.assert_called_once_with( @@ -1077,7 +1077,7 @@ def test_lookup_bucket_miss_w_defaults(self): self.assertIsNone(bucket) - expected_path = "/b/%s" % (bucket_name,) + expected_path = f"/b/{bucket_name}" expected_query_params = {"projection": "noAcl"} expected_headers = {} client._get_resource.assert_called_once_with( @@ -1109,7 +1109,7 @@ def test_lookup_bucket_hit_w_timeout(self): self.assertIsInstance(bucket, Bucket) self.assertEqual(bucket.name, bucket_name) - expected_path = "/b/%s" % (bucket_name,) + expected_path = f"/b/{bucket_name}" expected_query_params = {"projection": "noAcl"} expected_headers = {} client._get_resource.assert_called_once_with( @@ -1139,7 +1139,7 @@ def test_lookup_bucket_hit_w_metageneration_match(self): self.assertIsInstance(bucket, Bucket) self.assertEqual(bucket.name, bucket_name) - expected_path = "/b/%s" % (bucket_name,) + expected_path = f"/b/{bucket_name}" expected_query_params = { "projection": "noAcl", "ifMetagenerationMatch": metageneration_number, @@ -1170,7 +1170,7 @@ def test_lookup_bucket_hit_w_retry(self): self.assertIsInstance(bucket, Bucket) self.assertEqual(bucket.name, bucket_name) - expected_path = "/b/%s" % (bucket_name,) + expected_path = f"/b/{bucket_name}" expected_query_params = {"projection": "noAcl"} expected_headers = {} client._get_resource.assert_called_once_with( @@ -1811,7 +1811,7 @@ def test_list_blobs_w_defaults_w_bucket_obj(self): self.assertIs(iterator.bucket, bucket) self.assertEqual(iterator.prefixes, set()) - expected_path = "/b/{}/o".format(bucket_name) + expected_path = f"/b/{bucket_name}/o" expected_item_to_value = _item_to_blob expected_page_token = None expected_max_results = None @@ -1855,7 +1855,7 @@ def test_list_blobs_w_explicit_w_user_project(self): bucket = client._bucket_arg_to_bucket.return_value = mock.Mock( spec=["path", "user_project"], ) - bucket.path = "/b/{}".format(bucket_name) + bucket.path = f"/b/{bucket_name}" bucket.user_project = user_project timeout = 42 retry = mock.Mock(spec=[]) @@ -1881,7 +1881,7 @@ def test_list_blobs_w_explicit_w_user_project(self): self.assertIs(iterator.bucket, bucket) self.assertEqual(iterator.prefixes, set()) - expected_path = "/b/{}/o".format(bucket_name) + expected_path = f"/b/{bucket_name}/o" expected_item_to_value = _item_to_blob expected_page_token = page_token expected_max_results = max_results @@ -2119,7 +2119,7 @@ def _create_hmac_key_helper( email = "storage-user-123@example.com" secret = "a" * 40 now = datetime.datetime.utcnow().replace(tzinfo=UTC) - now_stamp = "{}Z".format(now.isoformat()) + now_stamp = f"{now.isoformat()}Z" if explicit_project is not None: expected_project = explicit_project @@ -2131,7 +2131,7 @@ def _create_hmac_key_helper( "metadata": { "accessId": access_id, "etag": "ETAG", - "id": "projects/{}/hmacKeys/{}".format(project, access_id), + "id": f"projects/{project}/hmacKeys/{access_id}", "project": expected_project, "state": "ACTIVE", "serviceAccountEmail": email, @@ -2170,7 +2170,7 @@ def _create_hmac_key_helper( self.assertEqual(metadata._properties, api_response["metadata"]) self.assertEqual(secret, api_response["secret"]) - expected_path = "/projects/{}/hmacKeys".format(expected_project) + expected_path = f"/projects/{expected_project}/hmacKeys" expected_data = None expected_query_params = {"serviceAccountEmail": email} @@ -2212,7 +2212,7 @@ def test_list_hmac_keys_w_defaults(self): self.assertIs(iterator, client._list_resource.return_value) - expected_path = "/projects/{}/hmacKeys".format(project) + expected_path = f"/projects/{project}/hmacKeys" expected_item_to_value = _item_to_hmac_key_metadata expected_max_results = None expected_extra_params = {} @@ -2252,7 +2252,7 @@ def test_list_hmac_keys_w_explicit(self): self.assertIs(iterator, client._list_resource.return_value) - expected_path = "/projects/{}/hmacKeys".format(other_project) + expected_path = f"/projects/{other_project}/hmacKeys" expected_item_to_value = _item_to_hmac_key_metadata expected_max_results = max_results expected_extra_params = { @@ -2300,7 +2300,7 @@ def test_get_hmac_key_metadata_wo_project(self): ) _, kwargs = http.request.call_args scheme, netloc, path, qs, _ = urllib.parse.urlsplit(kwargs.get("url")) - self.assertEqual("%s://%s" % (scheme, netloc), client._connection.API_BASE_URL) + self.assertEqual(f"{scheme}://{netloc}", client._connection.API_BASE_URL) self.assertEqual( path, "/".join( @@ -2355,7 +2355,7 @@ def test_get_hmac_key_metadata_w_project(self): ) _, kwargs = http.request.call_args scheme, netloc, path, qs, _ = urllib.parse.urlsplit(kwargs.get("url")) - self.assertEqual("%s://%s" % (scheme, netloc), client._connection.API_BASE_URL) + self.assertEqual(f"{scheme}://{netloc}", client._connection.API_BASE_URL) self.assertEqual( path, "/".join( @@ -2515,7 +2515,7 @@ def test_get_signed_policy_v4_virtual_hosted_style(self): credentials=_create_signing_credentials(), ) self.assertEqual( - policy["url"], "https://{}.storage.googleapis.com/".format(BUCKET_NAME) + policy["url"], f"https://{BUCKET_NAME}.storage.googleapis.com/" ) def test_get_signed_policy_v4_bucket_bound_hostname(self): diff --git a/tests/unit/test_hmac_key.py b/tests/unit/test_hmac_key.py index 59a2b221f..917006b96 100644 --- a/tests/unit/test_hmac_key.py +++ b/tests/unit/test_hmac_key.py @@ -177,7 +177,7 @@ def test_time_created_getter(self): metadata = self._make_one() now = datetime.datetime.utcnow() - now_stamp = "{}Z".format(now.isoformat()) + now_stamp = f"{now.isoformat()}Z" metadata._properties["timeCreated"] = now_stamp self.assertEqual(metadata.time_created, now.replace(tzinfo=UTC)) @@ -187,7 +187,7 @@ def test_updated_getter(self): metadata = self._make_one() now = datetime.datetime.utcnow() - now_stamp = "{}Z".format(now.isoformat()) + now_stamp = f"{now.isoformat()}Z" metadata._properties["updated"] = now_stamp self.assertEqual(metadata.updated, now.replace(tzinfo=UTC)) @@ -203,9 +203,7 @@ def test_path_w_access_id_wo_project(self): metadata = self._make_one() metadata._properties["accessId"] = access_id - expected_path = "/projects/{}/hmacKeys/{}".format( - client.DEFAULT_PROJECT, access_id - ) + expected_path = f"/projects/{client.DEFAULT_PROJECT}/hmacKeys/{access_id}" self.assertEqual(metadata.path, expected_path) def test_path_w_access_id_w_explicit_project(self): @@ -215,7 +213,7 @@ def test_path_w_access_id_w_explicit_project(self): metadata._properties["accessId"] = access_id metadata._properties["projectId"] = project - expected_path = "/projects/{}/hmacKeys/{}".format(project, access_id) + expected_path = f"/projects/{project}/hmacKeys/{access_id}" self.assertEqual(metadata.path, expected_path) def test_exists_miss_w_defaults(self): @@ -231,7 +229,7 @@ def test_exists_miss_w_defaults(self): self.assertFalse(metadata.exists()) - expected_path = "/projects/{}/hmacKeys/{}".format(project, access_id) + expected_path = f"/projects/{project}/hmacKeys/{access_id}" expected_query_params = {} client._get_resource.assert_called_once_with( expected_path, @@ -260,7 +258,7 @@ def test_exists_hit_w_explicit_w_user_project(self): self.assertTrue(metadata.exists(timeout=timeout, retry=retry)) - expected_path = "/projects/{}/hmacKeys/{}".format(project, access_id) + expected_path = f"/projects/{project}/hmacKeys/{access_id}" expected_query_params = {"userProject": user_project} client._get_resource.assert_called_once_with( expected_path, @@ -283,7 +281,7 @@ def test_reload_miss_w_defaults(self): with self.assertRaises(NotFound): metadata.reload() - expected_path = "/projects/{}/hmacKeys/{}".format(project, access_id) + expected_path = f"/projects/{project}/hmacKeys/{access_id}" expected_query_params = {} client._get_resource.assert_called_once_with( expected_path, @@ -314,7 +312,7 @@ def test_reload_hit_w_project_set(self): self.assertEqual(metadata._properties, resource) - expected_path = "/projects/{}/hmacKeys/{}".format(project, access_id) + expected_path = f"/projects/{project}/hmacKeys/{access_id}" expected_query_params = {"userProject": user_project} client._get_resource.assert_called_once_with( expected_path, @@ -338,7 +336,7 @@ def test_update_miss_no_project_set_w_defaults(self): with self.assertRaises(NotFound): metadata.update() - expected_path = "/projects/{}/hmacKeys/{}".format(project, access_id) + expected_path = f"/projects/{project}/hmacKeys/{access_id}" expected_data = {"state": "INACTIVE"} expected_query_params = {} client._put_resource.assert_called_once_with( @@ -373,7 +371,7 @@ def test_update_hit_w_project_set_w_timeout_w_retry(self): self.assertEqual(metadata._properties, resource) - expected_path = "/projects/{}/hmacKeys/{}".format(project, access_id) + expected_path = f"/projects/{project}/hmacKeys/{access_id}" expected_data = {"state": "ACTIVE"} expected_query_params = {"userProject": user_project} client._put_resource.assert_called_once_with( @@ -411,7 +409,7 @@ def test_delete_miss_no_project_set_w_defaults(self): with self.assertRaises(NotFound): metadata.delete() - expected_path = "/projects/{}/hmacKeys/{}".format(client.project, access_id) + expected_path = f"/projects/{client.project}/hmacKeys/{access_id}" expected_query_params = {} client._delete_resource.assert_called_once_with( expected_path, @@ -436,7 +434,7 @@ def test_delete_hit_w_project_set_w_explicit_timeout_retry(self): metadata.delete(timeout=timeout, retry=retry) - expected_path = "/projects/{}/hmacKeys/{}".format(project, access_id) + expected_path = f"/projects/{project}/hmacKeys/{access_id}" expected_query_params = {"userProject": user_project} client._delete_resource.assert_called_once_with( expected_path, diff --git a/tests/unit/test_notification.py b/tests/unit/test_notification.py index cf4e15c13..e5f07d5c7 100644 --- a/tests/unit/test_notification.py +++ b/tests/unit/test_notification.py @@ -33,10 +33,8 @@ class TestBucketNotification(unittest.TestCase): NOTIFICATION_ID = "123" SELF_LINK = "https://example.com/notification/123" ETAG = "DEADBEEF" - CREATE_PATH = "/b/{}/notificationConfigs".format(BUCKET_NAME) - NOTIFICATION_PATH = "/b/{}/notificationConfigs/{}".format( - BUCKET_NAME, NOTIFICATION_ID - ) + CREATE_PATH = f"/b/{BUCKET_NAME}/notificationConfigs" + NOTIFICATION_PATH = f"/b/{BUCKET_NAME}/notificationConfigs/{NOTIFICATION_ID}" @staticmethod def event_types(): From b0bf411f8fec8712b3eeb99a2dd33de6d82312f8 Mon Sep 17 00:00:00 2001 From: Rebecca Peterson <44721098+rebecca-pete@users.noreply.github.com> Date: Fri, 20 May 2022 10:54:14 -0700 Subject: [PATCH 16/28] docs(samples): Update the Recovery Point Objective (RPO) sample output (#725) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit @ddelgrosso1 Related to b/217259317. Thank you for opening a Pull Request! Before submitting your PR, there are a few things you can do to make sure it goes smoothly: - [ ] Make sure to open an issue as a [bug/issue](https://github.com/googleapis/python-storage/issues/new/choose) before writing your code! That way we can discuss the change, evaluate designs, and agree on the general idea - [ ] Ensure the tests and linter pass - [ ] Code coverage does not decrease (if any source code was changed) - [ ] Appropriate docs were updated (if necessary) Fixes # 🦕 --- google/cloud/storage/constants.py | 6 ++++-- samples/snippets/rpo_test.py | 6 +++--- samples/snippets/storage_create_bucket_turbo_replication.py | 2 +- samples/snippets/storage_set_rpo_async_turbo.py | 2 +- samples/snippets/storage_set_rpo_default.py | 4 ++-- 5 files changed, 11 insertions(+), 9 deletions(-) diff --git a/google/cloud/storage/constants.py b/google/cloud/storage/constants.py index b8ac87886..babbc5a42 100644 --- a/google/cloud/storage/constants.py +++ b/google/cloud/storage/constants.py @@ -119,13 +119,15 @@ """ RPO_ASYNC_TURBO = "ASYNC_TURBO" -"""Turbo Replication RPO +"""The recovery point objective (RPO) indicates how quickly newly written objects are asynchronously replicated to a separate geographic location. +When the RPO value is set to ASYNC_TURBO, the turbo replication feature is enabled. See: https://cloud.google.com/storage/docs/managing-turbo-replication """ RPO_DEFAULT = "DEFAULT" -"""Default RPO +"""The recovery point objective (RPO) indicates how quickly newly written objects are asynchronously replicated to a separate geographic location. +When the RPO value is set to DEFAULT, the default replication behavior is enabled. See: https://cloud.google.com/storage/docs/managing-turbo-replication """ diff --git a/samples/snippets/rpo_test.py b/samples/snippets/rpo_test.py index f1f16e7fb..befc0334a 100644 --- a/samples/snippets/rpo_test.py +++ b/samples/snippets/rpo_test.py @@ -45,17 +45,17 @@ def test_get_rpo(dual_region_bucket, capsys): def test_set_rpo_async_turbo(dual_region_bucket, capsys): storage_set_rpo_async_turbo.set_rpo_async_turbo(dual_region_bucket.name) out, _ = capsys.readouterr() - assert f"RPO is ASYNC_TURBO for {dual_region_bucket.name}." in out + assert f"RPO is set to ASYNC_TURBO for {dual_region_bucket.name}." in out def test_set_rpo_default(dual_region_bucket, capsys): storage_set_rpo_default.set_rpo_default(dual_region_bucket.name) out, _ = capsys.readouterr() - assert f"RPO is DEFAULT for {dual_region_bucket.name}." in out + assert f"RPO is set to DEFAULT for {dual_region_bucket.name}." in out def test_create_bucket_turbo_replication(capsys): bucket_name = f"test-rpo-{uuid.uuid4()}" storage_create_bucket_turbo_replication.create_bucket_turbo_replication(bucket_name) out, _ = capsys.readouterr() - assert f"{bucket_name} created with RPO ASYNC_TURBO in NAM4." in out + assert f"{bucket_name} created with the recovery point objective (RPO) set to ASYNC_TURBO in NAM4." in out diff --git a/samples/snippets/storage_create_bucket_turbo_replication.py b/samples/snippets/storage_create_bucket_turbo_replication.py index 68f0ba482..3d26616ec 100644 --- a/samples/snippets/storage_create_bucket_turbo_replication.py +++ b/samples/snippets/storage_create_bucket_turbo_replication.py @@ -39,7 +39,7 @@ def create_bucket_turbo_replication(bucket_name): bucket.rpo = RPO_ASYNC_TURBO bucket.create() - print(f"{bucket.name} created with RPO {bucket.rpo} in {bucket.location}.") + print(f"{bucket.name} created with the recovery point objective (RPO) set to {bucket.rpo} in {bucket.location}.") # [END storage_create_bucket_turbo_replication] diff --git a/samples/snippets/storage_set_rpo_async_turbo.py b/samples/snippets/storage_set_rpo_async_turbo.py index 10b4c67a3..a351cb8f8 100644 --- a/samples/snippets/storage_set_rpo_async_turbo.py +++ b/samples/snippets/storage_set_rpo_async_turbo.py @@ -39,7 +39,7 @@ def set_rpo_async_turbo(bucket_name): bucket.rpo = RPO_ASYNC_TURBO bucket.patch() - print(f"RPO is ASYNC_TURBO for {bucket.name}.") + print(f"RPO is set to ASYNC_TURBO for {bucket.name}.") # [END storage_set_rpo_async_turbo] diff --git a/samples/snippets/storage_set_rpo_default.py b/samples/snippets/storage_set_rpo_default.py index 8d41b1fe0..883fee0c9 100644 --- a/samples/snippets/storage_set_rpo_default.py +++ b/samples/snippets/storage_set_rpo_default.py @@ -16,7 +16,7 @@ import sys -"""Sample that sets RPO (Recovery Point Objective) to default +"""Sample that sets the replication behavior or recovery point objective (RPO) to default. This sample is used on this page: https://cloud.google.com/storage/docs/managing-turbo-replication For more information, see README.md. @@ -39,7 +39,7 @@ def set_rpo_default(bucket_name): bucket.rpo = RPO_DEFAULT bucket.patch() - print(f"RPO is DEFAULT for {bucket.name}.") + print(f"RPO is set to DEFAULT for {bucket.name}.") # [END storage_set_rpo_default] From 7ec6f4d0bb11f86a184e5bf1099464d98b24796c Mon Sep 17 00:00:00 2001 From: cojenco Date: Thu, 26 May 2022 15:18:54 -0700 Subject: [PATCH 17/28] chore: fix requests intersphinx url (#801) --- docs/conf.py | 4 +++- docs/retry_timeout.rst | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 7a2f13fca..082c4a145 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -368,7 +368,9 @@ "grpc": ("https://grpc.github.io/grpc/python/", None), "proto-plus": ("https://proto-plus-python.readthedocs.io/en/latest/", None), "protobuf": ("https://googleapis.dev/python/protobuf/latest/", None), - "requests": ("https://docs.python-requests.org/en/master/", None), + # python-requests url temporary change related to + # https://github.com/psf/requests/issues/6140#issuecomment-1135071992 + "python-requests": ("https://requests.readthedocs.io/en/stable/", None), } diff --git a/docs/retry_timeout.rst b/docs/retry_timeout.rst index db072013b..18e25304a 100644 --- a/docs/retry_timeout.rst +++ b/docs/retry_timeout.rst @@ -47,7 +47,7 @@ in your code, using one of three forms: See also: - :ref:`Timeouts in requests ` + `Timeouts in requests `_ .. _configuring_retries: From 32ed45f564f602fc03d45a2938ea9efb596e0421 Mon Sep 17 00:00:00 2001 From: cojenco Date: Thu, 26 May 2022 17:14:14 -0700 Subject: [PATCH 18/28] chore: pin protobuf (#799) --- setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.py b/setup.py index af6c97fb2..8686745f7 100644 --- a/setup.py +++ b/setup.py @@ -33,9 +33,8 @@ "google-cloud-core >= 2.3.0, < 3.0dev", "google-resumable-media >= 2.3.2", "requests >= 2.18.0, < 3.0.0dev", - "protobuf", ] -extras = {} +extras = {"protobuf": ["protobuf<5.0.0dev"]} # Setup boilerplate below this line. From 18bbbd32194e88bdcea3b902ba6a0e5cee0a42c6 Mon Sep 17 00:00:00 2001 From: cojenco Date: Fri, 27 May 2022 14:31:00 -0700 Subject: [PATCH 19/28] chore: update sphinx arrangement to remove duplicate entries on cloudsite (#790) * chore: move module docs under storage * cleanup acl docstring and remove incorrect tags * merge conflict --- docs/acl.rst | 6 -- docs/index.rst | 13 +-- docs/storage/acl.rst | 89 +++++++++++++++++++ docs/{ => storage}/batch.rst | 0 docs/{ => storage}/blobs.rst | 0 docs/{ => storage}/buckets.rst | 0 docs/{ => storage}/client.rst | 0 docs/{ => storage}/constants.rst | 0 docs/{ => storage}/fileio.rst | 0 .../generation_metageneration.rst | 0 docs/{ => storage}/hmac_key.rst | 0 docs/storage/modules.rst | 17 ++++ docs/{ => storage}/notification.rst | 0 docs/{ => storage}/retry.rst | 0 docs/{ => storage}/retry_timeout.rst | 0 docs/{ => storage}/snippets.py | 84 ++++++++--------- google/cloud/storage/__init__.py | 4 +- google/cloud/storage/acl.py | 72 +-------------- google/cloud/storage/blob.py | 8 +- google/cloud/storage/bucket.py | 32 +++---- google/cloud/storage/client.py | 16 ++-- 21 files changed, 180 insertions(+), 161 deletions(-) delete mode 100644 docs/acl.rst create mode 100644 docs/storage/acl.rst rename docs/{ => storage}/batch.rst (100%) rename docs/{ => storage}/blobs.rst (100%) rename docs/{ => storage}/buckets.rst (100%) rename docs/{ => storage}/client.rst (100%) rename docs/{ => storage}/constants.rst (100%) rename docs/{ => storage}/fileio.rst (100%) rename docs/{ => storage}/generation_metageneration.rst (100%) rename docs/{ => storage}/hmac_key.rst (100%) create mode 100644 docs/storage/modules.rst rename docs/{ => storage}/notification.rst (100%) rename docs/{ => storage}/retry.rst (100%) rename docs/{ => storage}/retry_timeout.rst (100%) rename docs/{ => storage}/snippets.py (86%) diff --git a/docs/acl.rst b/docs/acl.rst deleted file mode 100644 index f1f7d0289..000000000 --- a/docs/acl.rst +++ /dev/null @@ -1,6 +0,0 @@ -ACL -~~~ - -.. automodule:: google.cloud.storage.acl - :members: - :show-inheritance: diff --git a/docs/index.rst b/docs/index.rst index 154c76d5e..5a9109944 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -13,18 +13,7 @@ API Reference .. toctree:: :maxdepth: 2 - client - blobs - buckets - acl - batch - fileio - constants - hmac_key - notification - retry - retry_timeout - generation_metageneration + storage/modules More Examples ------------- diff --git a/docs/storage/acl.rst b/docs/storage/acl.rst new file mode 100644 index 000000000..f96cd6597 --- /dev/null +++ b/docs/storage/acl.rst @@ -0,0 +1,89 @@ +ACL +=== + +Cloud Storage uses access control lists (ACLs) to manage object and bucket access. +ACLs are the mechanism you use to share files with other users and allow +other users to access your buckets and files. + +ACLs are suitable for fine-grained control, but you may prefer using IAM to +control access at the project level. See also: +`Cloud Storage Control Access to Data `_ + + +:class:`google.cloud.storage.bucket.Bucket` has a getting method that creates +an ACL object under the hood, and you can interact with that using +:func:`google.cloud.storage.bucket.Bucket.acl`: + +.. code-block:: python + + client = storage.Client() + bucket = client.get_bucket(bucket_name) + acl = bucket.acl + +Adding and removing permissions can be done with the following methods +(in increasing order of granularity): + +- :func:`ACL.all` + corresponds to access for all users. +- :func:`ACL.all_authenticated` corresponds + to access for all users that are signed into a Google account. +- :func:`ACL.domain` corresponds to access on a + per Google Apps domain (ie, ``example.com``). +- :func:`ACL.group` corresponds to access on a + per group basis (either by ID or e-mail address). +- :func:`ACL.user` corresponds to access on a + per user basis (either by ID or e-mail address). + +And you are able to ``grant`` and ``revoke`` the following roles: + +- **Reading**: + :func:`_ACLEntity.grant_read` and :func:`_ACLEntity.revoke_read` +- **Writing**: + :func:`_ACLEntity.grant_write` and :func:`_ACLEntity.revoke_write` +- **Owning**: + :func:`_ACLEntity.grant_owner` and :func:`_ACLEntity.revoke_owner` + +You can use any of these like any other factory method (these happen to +be :class:`_ACLEntity` factories): + +.. code-block:: python + + acl.user("me@example.org").grant_read() + acl.all_authenticated().grant_write() + +After that, you can save any changes you make with the +:func:`google.cloud.storage.acl.ACL.save` method: + +.. code-block:: python + + acl.save() + + +You can alternatively save any existing :class:`google.cloud.storage.acl.ACL` +object (whether it was created by a factory method or not) from a +:class:`google.cloud.storage.bucket.Bucket`: + +.. code-block:: python + + bucket.acl.save(acl=acl) + + +To get the list of ``entity`` and ``role`` for each unique pair, the +:class:`ACL` class is iterable: + +.. code-block:: python + + print(list(acl)) + # [{'role': 'OWNER', 'entity': 'allUsers'}, ...] + + +This list of tuples can be used as the ``entity`` and ``role`` fields +when sending metadata for ACLs to the API. + + +ACL Module +---------- + +.. automodule:: google.cloud.storage.acl + :members: + :show-inheritance: diff --git a/docs/batch.rst b/docs/storage/batch.rst similarity index 100% rename from docs/batch.rst rename to docs/storage/batch.rst diff --git a/docs/blobs.rst b/docs/storage/blobs.rst similarity index 100% rename from docs/blobs.rst rename to docs/storage/blobs.rst diff --git a/docs/buckets.rst b/docs/storage/buckets.rst similarity index 100% rename from docs/buckets.rst rename to docs/storage/buckets.rst diff --git a/docs/client.rst b/docs/storage/client.rst similarity index 100% rename from docs/client.rst rename to docs/storage/client.rst diff --git a/docs/constants.rst b/docs/storage/constants.rst similarity index 100% rename from docs/constants.rst rename to docs/storage/constants.rst diff --git a/docs/fileio.rst b/docs/storage/fileio.rst similarity index 100% rename from docs/fileio.rst rename to docs/storage/fileio.rst diff --git a/docs/generation_metageneration.rst b/docs/storage/generation_metageneration.rst similarity index 100% rename from docs/generation_metageneration.rst rename to docs/storage/generation_metageneration.rst diff --git a/docs/hmac_key.rst b/docs/storage/hmac_key.rst similarity index 100% rename from docs/hmac_key.rst rename to docs/storage/hmac_key.rst diff --git a/docs/storage/modules.rst b/docs/storage/modules.rst new file mode 100644 index 000000000..9148a4385 --- /dev/null +++ b/docs/storage/modules.rst @@ -0,0 +1,17 @@ +Modules for Python Storage +-------------------------- +.. toctree:: + :maxdepth: 2 + + client + blobs + buckets + acl + batch + fileio + constants + hmac_key + notification + retry + retry_timeout + generation_metageneration \ No newline at end of file diff --git a/docs/notification.rst b/docs/storage/notification.rst similarity index 100% rename from docs/notification.rst rename to docs/storage/notification.rst diff --git a/docs/retry.rst b/docs/storage/retry.rst similarity index 100% rename from docs/retry.rst rename to docs/storage/retry.rst diff --git a/docs/retry_timeout.rst b/docs/storage/retry_timeout.rst similarity index 100% rename from docs/retry_timeout.rst rename to docs/storage/retry_timeout.rst diff --git a/docs/snippets.py b/docs/storage/snippets.py similarity index 86% rename from docs/snippets.py rename to docs/storage/snippets.py index 7ee3a62a0..93884900f 100644 --- a/docs/snippets.py +++ b/docs/storage/snippets.py @@ -34,7 +34,7 @@ def snippet(func): @snippet def storage_get_started(to_delete): - # [START storage_get_started] + # START storage_get_started client = storage.Client() bucket = client.get_bucket("bucket-id-here") # Then do other things... @@ -43,7 +43,7 @@ def storage_get_started(to_delete): blob.upload_from_string("New contents!") blob2 = bucket.blob("/remote/path/storage.txt") blob2.upload_from_filename(filename="/local/path.txt") - # [END storage_get_started] + # END storage_get_started to_delete.append(bucket) @@ -53,40 +53,40 @@ def client_bucket_acl(client, to_delete): bucket_name = "system-test-bucket" client.create_bucket(bucket_name) - # [START client_bucket_acl] + # START client_bucket_acl client = storage.Client() bucket = client.get_bucket(bucket_name) acl = bucket.acl - # [END client_bucket_acl] + # END client_bucket_acl to_delete.append(bucket) - # [START acl_user_settings] + # START acl_user_settings acl.user("me@example.org").grant_read() acl.all_authenticated().grant_write() - # [END acl_user_settings] + # END acl_user_settings - # [START acl_save] + # START acl_save acl.save() - # [END acl_save] + # END acl_save - # [START acl_revoke_write] + # START acl_revoke_write acl.all().grant_read() acl.all().revoke_write() - # [END acl_revoke_write] + # END acl_revoke_write - # [START acl_save_bucket] + # START acl_save_bucket bucket.acl.save(acl=acl) - # [END acl_save_bucket] + # END acl_save_bucket - # [START acl_print] + # START acl_print print(list(acl)) # [{'role': 'OWNER', 'entity': 'allUsers'}, ...] - # [END acl_print] + # END acl_print @snippet def download_to_file(to_delete): - # [START download_to_file] + # START download_to_file from google.cloud.storage import Blob client = storage.Client(project="my-project") @@ -96,14 +96,14 @@ def download_to_file(to_delete): blob.upload_from_string("my secret message.") with open("/tmp/my-secure-file", "wb") as file_obj: client.download_to_file(blob, file_obj) - # [END download_to_file] + # END download_to_file to_delete.append(blob) @snippet def upload_from_file(to_delete): - # [START upload_from_file] + # START upload_from_file from google.cloud.storage import Blob client = storage.Client(project="my-project") @@ -112,7 +112,7 @@ def upload_from_file(to_delete): blob = Blob("secure-data", bucket, encryption_key=encryption_key) with open("my-file", "rb") as my_file: blob.upload_from_file(my_file) - # [END upload_from_file] + # END upload_from_file to_delete.append(blob) @@ -121,21 +121,21 @@ def upload_from_file(to_delete): def get_blob(to_delete): from google.cloud.storage.blob import Blob - # [START get_blob] + # START get_blob client = storage.Client() bucket = client.get_bucket("my-bucket") assert isinstance(bucket.get_blob("/path/to/blob.txt"), Blob) # assert not bucket.get_blob("/does-not-exist.txt") # None - # [END get_blob] + # END get_blob to_delete.append(bucket) @snippet def delete_blob(to_delete): - # [START delete_blob] + # START delete_blob from google.cloud.exceptions import NotFound client = storage.Client() @@ -148,12 +148,12 @@ def delete_blob(to_delete): bucket.delete_blob("doesnt-exist") except NotFound: pass - # [END delete_blob] + # END delete_blob blob = None - # [START delete_blobs] + # START delete_blobs bucket.delete_blobs([blob], on_error=lambda blob: None) - # [END delete_blobs] + # END delete_blobs to_delete.append(bucket) @@ -161,15 +161,15 @@ def delete_blob(to_delete): @snippet def configure_website(to_delete): bucket_name = "test-bucket" - # [START configure_website] + # START configure_website client = storage.Client() bucket = client.get_bucket(bucket_name) bucket.configure_website("index.html", "404.html") - # [END configure_website] + # END configure_website - # [START make_public] + # START make_public bucket.make_public(recursive=True, future=True) - # [END make_public] + # END make_public to_delete.append(bucket) @@ -178,34 +178,34 @@ def configure_website(to_delete): def get_bucket(client, to_delete): import google - # [START get_bucket] + # START get_bucket try: bucket = client.get_bucket("my-bucket") except google.cloud.exceptions.NotFound: print("Sorry, that bucket does not exist!") - # [END get_bucket] + # END get_bucket to_delete.append(bucket) @snippet def add_lifecycle_delete_rule(client, to_delete): - # [START add_lifecycle_delete_rule] + # START add_lifecycle_delete_rule bucket = client.get_bucket("my-bucket") bucket.add_lifecycle_delete_rule(age=2) bucket.patch() - # [END add_lifecycle_delete_rule] + # END add_lifecycle_delete_rule to_delete.append(bucket) @snippet def add_lifecycle_set_storage_class_rule(client, to_delete): - # [START add_lifecycle_set_storage_class_rule] + # START add_lifecycle_set_storage_class_rule bucket = client.get_bucket("my-bucket") bucket.add_lifecycle_set_storage_class_rule( "COLD_LINE", matches_storage_class=["NEARLINE"] ) bucket.patch() - # [END add_lifecycle_set_storage_class_rule] + # END add_lifecycle_set_storage_class_rule to_delete.append(bucket) @@ -213,14 +213,14 @@ def add_lifecycle_set_storage_class_rule(client, to_delete): def lookup_bucket(client, to_delete): from google.cloud.storage.bucket import Bucket - # [START lookup_bucket] + # START lookup_bucket bucket = client.lookup_bucket("doesnt-exist") assert not bucket # None bucket = client.lookup_bucket("my-bucket") assert isinstance(bucket, Bucket) # - # [END lookup_bucket] + # END lookup_bucket to_delete.append(bucket) @@ -229,21 +229,21 @@ def lookup_bucket(client, to_delete): def create_bucket(client, to_delete): from google.cloud.storage import Bucket - # [START create_bucket] + # START create_bucket bucket = client.create_bucket("my-bucket") assert isinstance(bucket, Bucket) # - # [END create_bucket] + # END create_bucket to_delete.append(bucket) @snippet def list_buckets(client, to_delete): - # [START list_buckets] + # START list_buckets for bucket in client.list_buckets(): print(bucket) - # [END list_buckets] + # END list_buckets for bucket in client.list_buckets(): to_delete.append(bucket) @@ -252,7 +252,7 @@ def list_buckets(client, to_delete): @snippet def policy_document(client): # pylint: disable=unused-argument - # [START policy_document] + # START policy_document bucket = client.bucket("my-bucket") conditions = [["starts-with", "$key", ""], {"acl": "public-read"}] @@ -277,7 +277,7 @@ def policy_document(client): ).format(bucket_name=bucket.name, policy_fields=policy_fields) print(upload_form) - # [END policy_document] + # END policy_document def _line_no(func): diff --git a/google/cloud/storage/__init__.py b/google/cloud/storage/__init__.py index b05efab8c..4e9c47f4a 100644 --- a/google/cloud/storage/__init__.py +++ b/google/cloud/storage/__init__.py @@ -17,8 +17,8 @@ You'll typically use these to get started with the API: .. literalinclude:: snippets.py - :start-after: [START storage_get_started] - :end-before: [END storage_get_started] + :start-after: START storage_get_started + :end-before: END storage_get_started :dedent: 4 The main concepts with this API are: diff --git a/google/cloud/storage/acl.py b/google/cloud/storage/acl.py index e876c152c..4458966ce 100644 --- a/google/cloud/storage/acl.py +++ b/google/cloud/storage/acl.py @@ -12,77 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Manipulate access control lists that Cloud Storage provides. - -:class:`google.cloud.storage.bucket.Bucket` has a getting method that creates -an ACL object under the hood, and you can interact with that using -:func:`google.cloud.storage.bucket.Bucket.acl`: - -.. literalinclude:: snippets.py - :start-after: [START client_bucket_acl] - :end-before: [END client_bucket_acl] - :dedent: 4 - - -Adding and removing permissions can be done with the following methods -(in increasing order of granularity): - -- :func:`ACL.all` - corresponds to access for all users. -- :func:`ACL.all_authenticated` corresponds - to access for all users that are signed into a Google account. -- :func:`ACL.domain` corresponds to access on a - per Google Apps domain (ie, ``example.com``). -- :func:`ACL.group` corresponds to access on a - per group basis (either by ID or e-mail address). -- :func:`ACL.user` corresponds to access on a - per user basis (either by ID or e-mail address). - -And you are able to ``grant`` and ``revoke`` the following roles: - -- **Reading**: - :func:`_ACLEntity.grant_read` and :func:`_ACLEntity.revoke_read` -- **Writing**: - :func:`_ACLEntity.grant_write` and :func:`_ACLEntity.revoke_write` -- **Owning**: - :func:`_ACLEntity.grant_owner` and :func:`_ACLEntity.revoke_owner` - -You can use any of these like any other factory method (these happen to -be :class:`_ACLEntity` factories): - -.. literalinclude:: snippets.py - :start-after: [START acl_user_settings] - :end-before: [END acl_user_settings] - :dedent: 4 - -After that, you can save any changes you make with the -:func:`google.cloud.storage.acl.ACL.save` method: - -.. literalinclude:: snippets.py - :start-after: [START acl_save] - :end-before: [END acl_save] - :dedent: 4 - -You can alternatively save any existing :class:`google.cloud.storage.acl.ACL` -object (whether it was created by a factory method or not) from a -:class:`google.cloud.storage.bucket.Bucket`: - -.. literalinclude:: snippets.py - :start-after: [START acl_save_bucket] - :end-before: [END acl_save_bucket] - :dedent: 4 - -To get the list of ``entity`` and ``role`` for each unique pair, the -:class:`ACL` class is iterable: - -.. literalinclude:: snippets.py - :start-after: [START acl_print] - :end-before: [END acl_print] - :dedent: 4 - -This list of tuples can be used as the ``entity`` and ``role`` fields -when sending metadata for ACLs to the API. -""" +"""Manage access to objects and buckets.""" from google.cloud.storage._helpers import _add_generation_match_parameters from google.cloud.storage.constants import _DEFAULT_TIMEOUT diff --git a/google/cloud/storage/blob.py b/google/cloud/storage/blob.py index a3ea714ef..752290b59 100644 --- a/google/cloud/storage/blob.py +++ b/google/cloud/storage/blob.py @@ -1041,8 +1041,8 @@ def download_to_file( encryption key: .. literalinclude:: snippets.py - :start-after: [START download_to_file] - :end-before: [END download_to_file] + :start-after: START download_to_file + :end-before: END download_to_file :dedent: 4 The ``encryption_key`` should be a str or bytes with a length of at @@ -2438,8 +2438,8 @@ def upload_from_file( [`customer-supplied`](https://cloud.google.com/storage/docs/encryption#customer-supplied) encryption key: .. literalinclude:: snippets.py - :start-after: [START upload_from_file] - :end-before: [END upload_from_file] + :start-after: START upload_from_file + :end-before: END upload_from_file :dedent: 4 The ``encryption_key`` should be a str or bytes with a length of at diff --git a/google/cloud/storage/bucket.py b/google/cloud/storage/bucket.py index 143629236..c98f005c3 100644 --- a/google/cloud/storage/bucket.py +++ b/google/cloud/storage/bucket.py @@ -1173,8 +1173,8 @@ def get_blob( This will return None if the blob doesn't exist: .. literalinclude:: snippets.py - :start-after: [START get_blob] - :end-before: [END get_blob] + :start-after: START get_blob + :end-before: END get_blob :dedent: 4 If :attr:`user_project` is set, bills the API request to that project. @@ -1592,8 +1592,8 @@ def delete_blob( For example: .. literalinclude:: snippets.py - :start-after: [START delete_blob] - :end-before: [END delete_blob] + :start-after: START delete_blob + :end-before: END delete_blob :dedent: 4 If :attr:`user_project` is set, bills the API request to that project. @@ -1640,8 +1640,8 @@ def delete_blob( ``on_error`` callback, e.g.: .. literalinclude:: snippets.py - :start-after: [START delete_blobs] - :end-before: [END delete_blobs] + :start-after: START delete_blobs + :end-before: END delete_blobs :dedent: 4 """ @@ -2311,8 +2311,8 @@ def add_lifecycle_delete_rule(self, **kw): https://cloud.google.com/storage/docs/json_api/v1/buckets .. literalinclude:: snippets.py - :start-after: [START add_lifecycle_delete_rule] - :end-before: [END add_lifecycle_delete_rule] + :start-after: START add_lifecycle_delete_rule + :end-before: END add_lifecycle_delete_rule :dedent: 4 :type kw: dict @@ -2329,8 +2329,8 @@ def add_lifecycle_set_storage_class_rule(self, storage_class, **kw): https://cloud.google.com/storage/docs/json_api/v1/buckets .. literalinclude:: snippets.py - :start-after: [START add_lifecycle_set_storage_class_rule] - :end-before: [END add_lifecycle_set_storage_class_rule] + :start-after: START add_lifecycle_set_storage_class_rule + :end-before: END add_lifecycle_set_storage_class_rule :dedent: 4 :type storage_class: str, one of :attr:`STORAGE_CLASSES`. @@ -2689,15 +2689,15 @@ def configure_website(self, main_page_suffix=None, not_found_page=None): of an index page and a page to use when a blob isn't found: .. literalinclude:: snippets.py - :start-after: [START configure_website] - :end-before: [END configure_website] + :start-after: START configure_website + :end-before: END configure_website :dedent: 4 You probably should also make the whole bucket public: .. literalinclude:: snippets.py - :start-after: [START make_public] - :end-before: [END make_public] + :start-after: START make_public + :end-before: END make_public :dedent: 4 This says: "Make the bucket public, and all the stuff already in @@ -3112,8 +3112,8 @@ def generate_upload_policy(self, conditions, expiration=None, client=None): For example: .. literalinclude:: snippets.py - :start-after: [START policy_document] - :end-before: [END policy_document] + :start-after: START policy_document + :end-before: END policy_document :dedent: 4 .. _policy documents: diff --git a/google/cloud/storage/client.py b/google/cloud/storage/client.py index f905e1be0..a22b70f9a 100644 --- a/google/cloud/storage/client.py +++ b/google/cloud/storage/client.py @@ -761,8 +761,8 @@ def get_bucket( Retrieve a bucket using a string. .. literalinclude:: snippets.py - :start-after: [START get_bucket] - :end-before: [END get_bucket] + :start-after: START get_bucket + :end-before: END get_bucket :dedent: 4 Get a bucket using a resource. @@ -802,8 +802,8 @@ def lookup_bucket( than catching an exception: .. literalinclude:: snippets.py - :start-after: [START lookup_bucket] - :end-before: [END lookup_bucket] + :start-after: START lookup_bucket + :end-before: END lookup_bucket :dedent: 4 :type bucket_name: str @@ -916,8 +916,8 @@ def create_bucket( Create a bucket using a string. .. literalinclude:: snippets.py - :start-after: [START create_bucket] - :end-before: [END create_bucket] + :start-after: START create_bucket + :end-before: END create_bucket :dedent: 4 Create a bucket using a resource. @@ -1333,8 +1333,8 @@ def list_buckets( bucket. .. literalinclude:: snippets.py - :start-after: [START list_buckets] - :end-before: [END list_buckets] + :start-after: START list_buckets + :end-before: END list_buckets :dedent: 4 This implements "storage.buckets.list". From 29bf346c5dfc017f344eb9231f7ef34cdcded5c8 Mon Sep 17 00:00:00 2001 From: cojenco Date: Wed, 1 Jun 2022 10:43:31 -0700 Subject: [PATCH 20/28] chore: update owlbot requests intersphinx url (#803) --- owlbot.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/owlbot.py b/owlbot.py index 42e3fdda8..2388b408c 100644 --- a/owlbot.py +++ b/owlbot.py @@ -34,7 +34,9 @@ "google-cloud-kms < 2.0dev", ], intersphinx_dependencies={ - "requests": "https://docs.python-requests.org/en/master/" + # python-requests url temporary change related to + # https://github.com/psf/requests/issues/6140#issuecomment-1135071992 + "requests": "https://requests.readthedocs.io/en/stable/" }, ) From 4dd0907b68e20d1ffcd0fe350831867197917e0d Mon Sep 17 00:00:00 2001 From: Dan Lee <71398022+dandhlee@users.noreply.github.com> Date: Wed, 1 Jun 2022 16:08:50 -0400 Subject: [PATCH 21/28] docs: fix changelog header to consistent size (#802) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * docs: fix changelog header to consistent size * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md Co-authored-by: Owl Bot Co-authored-by: cojenco --- CHANGELOG.md | 24 ++++++++++++------------ docs/conf.py | 4 +--- samples/snippets/noxfile.py | 4 ++-- 3 files changed, 15 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e9c28050d..58fb809de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,7 +22,7 @@ * fix links in blob module ([#759](https://github.com/googleapis/python-storage/issues/759)) ([9b29314](https://github.com/googleapis/python-storage/commit/9b2931430b0796ffb23ec4efacd82dacad36f40f)) -### [2.2.1](https://github.com/googleapis/python-storage/compare/v2.2.0...v2.2.1) (2022-03-15) +## [2.2.1](https://github.com/googleapis/python-storage/compare/v2.2.0...v2.2.1) (2022-03-15) ### Bug Fixes @@ -105,7 +105,7 @@ * add README to samples subdirectory ([#639](https://www.github.com/googleapis/python-storage/issues/639)) ([58af882](https://www.github.com/googleapis/python-storage/commit/58af882c047c31f59486513c568737082bca6350)) * update samples readme with cli args ([#651](https://www.github.com/googleapis/python-storage/issues/651)) ([75dda81](https://www.github.com/googleapis/python-storage/commit/75dda810e808074d18dfe7915f1403ad01bf2f02)) -### [1.42.3](https://www.github.com/googleapis/python-storage/compare/v1.42.2...v1.42.3) (2021-09-30) +## [1.42.3](https://www.github.com/googleapis/python-storage/compare/v1.42.2...v1.42.3) (2021-09-30) ### Bug Fixes @@ -114,7 +114,7 @@ * check response code in batch.finish ([#609](https://www.github.com/googleapis/python-storage/issues/609)) ([318a286](https://www.github.com/googleapis/python-storage/commit/318a286d709427bfe9f3a37e933c255ac51b3033)) * skip tests that use unspecified pap until we get the change in ([#600](https://www.github.com/googleapis/python-storage/issues/600)) ([38b9b55](https://www.github.com/googleapis/python-storage/commit/38b9b5582e2c6bbd1acab2b49410084170466fad)) -### [1.42.2](https://www.github.com/googleapis/python-storage/compare/v1.42.1...v1.42.2) (2021-09-16) +## [1.42.2](https://www.github.com/googleapis/python-storage/compare/v1.42.1...v1.42.2) (2021-09-16) ### Bug Fixes @@ -123,7 +123,7 @@ * add unpinned protobuf for python3 ([#592](https://www.github.com/googleapis/python-storage/issues/592)) ([53f7ad0](https://www.github.com/googleapis/python-storage/commit/53f7ad0204ad425011da9162d1a78f8276c837eb)) * pin six as a required dependency ([#589](https://www.github.com/googleapis/python-storage/issues/589)) ([9ca97bf](https://www.github.com/googleapis/python-storage/commit/9ca97bf9139c71cd033c78af73da904b27d8ff50)) -### [1.42.1](https://www.github.com/googleapis/python-storage/compare/v1.42.0...v1.42.1) (2021-09-07) +## [1.42.1](https://www.github.com/googleapis/python-storage/compare/v1.42.0...v1.42.1) (2021-09-07) ### Bug Fixes @@ -154,7 +154,7 @@ * update supported / removed Python versions in README ([#519](https://www.github.com/googleapis/python-storage/issues/519)) ([1f1b138](https://www.github.com/googleapis/python-storage/commit/1f1b138865fb171535ee0cf768aff1987ed58914)) -### [1.41.1](https://www.github.com/googleapis/python-storage/compare/v1.41.0...v1.41.1) (2021-07-20) +## [1.41.1](https://www.github.com/googleapis/python-storage/compare/v1.41.0...v1.41.1) (2021-07-20) ### Bug Fixes @@ -232,7 +232,7 @@ * revise docstrings for generate_signed_url ([#408](https://www.github.com/googleapis/python-storage/issues/408)) ([f090548](https://www.github.com/googleapis/python-storage/commit/f090548437142b635191e90dcee1acd4c38e565c)) -### [1.37.1](https://www.github.com/googleapis/python-storage/compare/v1.37.0...v1.37.1) (2021-04-02) +## [1.37.1](https://www.github.com/googleapis/python-storage/compare/v1.37.0...v1.37.1) (2021-04-02) ### Bug Fixes @@ -252,14 +252,14 @@ * update user_project usage and documentation in bucket/client class methods ([#396](https://www.github.com/googleapis/python-storage/issues/396)) ([1a2734b](https://www.github.com/googleapis/python-storage/commit/1a2734ba6d316ce51e4e141571331e86196462b9)) -### [1.36.2](https://www.github.com/googleapis/python-storage/compare/v1.36.1...v1.36.2) (2021-03-09) +## [1.36.2](https://www.github.com/googleapis/python-storage/compare/v1.36.1...v1.36.2) (2021-03-09) ### Bug Fixes * update batch connection to request api endpoint info from client ([#392](https://www.github.com/googleapis/python-storage/issues/392)) ([91fc6d9](https://www.github.com/googleapis/python-storage/commit/91fc6d9870a36308b15a827ed6a691e5b4669b62)) -### [1.36.1](https://www.github.com/googleapis/python-storage/compare/v1.36.0...v1.36.1) (2021-02-19) +## [1.36.1](https://www.github.com/googleapis/python-storage/compare/v1.36.0...v1.36.1) (2021-02-19) ### Bug Fixes @@ -283,7 +283,7 @@ * pass the unused parameter ([#349](https://www.github.com/googleapis/python-storage/issues/349)) ([5c60d24](https://www.github.com/googleapis/python-storage/commit/5c60d240aa98d2a1dcc6933d6da2ce60ea1b7559)) * set custom_time on uploads ([#374](https://www.github.com/googleapis/python-storage/issues/374)) ([f048be1](https://www.github.com/googleapis/python-storage/commit/f048be10416f51cea4e6c8c5b805df7b5d9c4d32)), closes [#372](https://www.github.com/googleapis/python-storage/issues/372) -### [1.35.1](https://www.github.com/googleapis/python-storage/compare/v1.35.0...v1.35.1) (2021-01-28) +## [1.35.1](https://www.github.com/googleapis/python-storage/compare/v1.35.0...v1.35.1) (2021-01-28) ### Bug Fixes @@ -342,14 +342,14 @@ * self-upload files for Unicode system test ([#296](https://www.github.com/googleapis/python-storage/issues/296)) ([6f865d9](https://www.github.com/googleapis/python-storage/commit/6f865d97a19278884356055dfeeaae92f7c63cc1)) * use version.py for versioning, avoid issues with discovering version via get_distribution ([#288](https://www.github.com/googleapis/python-storage/issues/288)) ([fcd1c4f](https://www.github.com/googleapis/python-storage/commit/fcd1c4f7c947eb95d6937783fd69670a570f145e)) -### [1.31.2](https://www.github.com/googleapis/python-storage/compare/v1.31.1...v1.31.2) (2020-09-23) +## [1.31.2](https://www.github.com/googleapis/python-storage/compare/v1.31.1...v1.31.2) (2020-09-23) ### Documentation * fix docstring example for 'blob.generate_signed_url' ([#278](https://www.github.com/googleapis/python-storage/issues/278)) ([2dc91c9](https://www.github.com/googleapis/python-storage/commit/2dc91c947e3693023b4478a15c460693808ea2d9)) -### [1.31.1](https://www.github.com/googleapis/python-storage/compare/v1.31.0...v1.31.1) (2020-09-16) +## [1.31.1](https://www.github.com/googleapis/python-storage/compare/v1.31.0...v1.31.1) (2020-09-16) ### Bug Fixes @@ -442,7 +442,7 @@ * fix upload object with bucket cmek enabled ([#158](https://www.github.com/googleapis/python-storage/issues/158)) ([5f27ffa](https://www.github.com/googleapis/python-storage/commit/5f27ffa3b1b55681453b594a0ef9e2811fc5f0c8)) * set default POST policy scheme to "http" ([#172](https://www.github.com/googleapis/python-storage/issues/172)) ([90c020d](https://www.github.com/googleapis/python-storage/commit/90c020d69a69ebc396416e4086a2e0838932130c)) -### [1.28.1](https://www.github.com/googleapis/python-storage/compare/v1.28.0...v1.28.1) (2020-04-28) +## [1.28.1](https://www.github.com/googleapis/python-storage/compare/v1.28.0...v1.28.1) (2020-04-28) ### Bug Fixes diff --git a/docs/conf.py b/docs/conf.py index 082c4a145..0e6ccdff0 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -368,9 +368,7 @@ "grpc": ("https://grpc.github.io/grpc/python/", None), "proto-plus": ("https://proto-plus-python.readthedocs.io/en/latest/", None), "protobuf": ("https://googleapis.dev/python/protobuf/latest/", None), - # python-requests url temporary change related to - # https://github.com/psf/requests/issues/6140#issuecomment-1135071992 - "python-requests": ("https://requests.readthedocs.io/en/stable/", None), + "requests": ("https://requests.readthedocs.io/en/stable/", None), } diff --git a/samples/snippets/noxfile.py b/samples/snippets/noxfile.py index 9f1cc8fb1..38bb0a572 100644 --- a/samples/snippets/noxfile.py +++ b/samples/snippets/noxfile.py @@ -66,7 +66,7 @@ sys.path.append(".") from noxfile_config import TEST_CONFIG_OVERRIDE except ImportError as e: - print(f"No user noxfile_config found: detail: {e}") + print("No user noxfile_config found: detail: {}".format(e)) TEST_CONFIG_OVERRIDE = {} # Update the TEST_CONFIG with the user supplied values. @@ -266,7 +266,7 @@ def py(session: nox.sessions.Session) -> None: _session_tests(session) else: session.skip( - f"SKIPPED: {session.python} tests are disabled for this sample." + "SKIPPED: {} tests are disabled for this sample.".format(session.python) ) From 9b3cbf3789c21462eac3c776cd29df12701e792f Mon Sep 17 00:00:00 2001 From: cojenco Date: Fri, 3 Jun 2022 09:47:21 -0700 Subject: [PATCH 22/28] fix: fix rewrite object in CMEK enabled bucket (#807) --- google/cloud/storage/blob.py | 10 +++++- tests/system/test_kms_integration.py | 11 ++++++ tests/unit/test_blob.py | 52 ++++++++++++++++++++++++++++ 3 files changed, 72 insertions(+), 1 deletion(-) diff --git a/google/cloud/storage/blob.py b/google/cloud/storage/blob.py index 752290b59..f47c09181 100644 --- a/google/cloud/storage/blob.py +++ b/google/cloud/storage/blob.py @@ -3576,7 +3576,15 @@ def rewrite( if source.generation: query_params["sourceGeneration"] = source.generation - if self.kms_key_name is not None: + # When a Customer Managed Encryption Key is used to encrypt Cloud Storage object + # at rest, object resource metadata will store the version of the Key Management + # Service cryptographic material. If a Blob instance with KMS Key metadata set is + # used to rewrite the object, then the existing kmsKeyName version + # value can't be used in the rewrite request and the client instead ignores it. + if ( + self.kms_key_name is not None + and "cryptoKeyVersions" not in self.kms_key_name + ): query_params["destinationKmsKeyName"] = self.kms_key_name _add_generation_match_parameters( diff --git a/tests/system/test_kms_integration.py b/tests/system/test_kms_integration.py index 9636acd54..87c1a7c07 100644 --- a/tests/system/test_kms_integration.py +++ b/tests/system/test_kms_integration.py @@ -224,6 +224,17 @@ def test_blob_rewrite_rotate_csek_to_cmek( assert dest.download_as_bytes() == source_data + # Test existing kmsKeyName version is ignored in the rewrite request + dest = kms_bucket.get_blob(blob_name) + source = kms_bucket.get_blob(blob_name) + token, rewritten, total = dest.rewrite(source) + + while token is not None: + token, rewritten, total = dest.rewrite(source, token=token) + + assert rewritten == len(source_data) + assert dest.download_as_bytes() == source_data + def test_blob_upload_w_bucket_cmek_enabled( kms_bucket, diff --git a/tests/unit/test_blob.py b/tests/unit/test_blob.py index cea384846..018ea4505 100644 --- a/tests/unit/test_blob.py +++ b/tests/unit/test_blob.py @@ -4942,6 +4942,58 @@ def test_rewrite_same_name_w_old_key_new_kms_key(self): _target_object=dest, ) + def test_rewrite_same_name_w_kms_key_w_version(self): + blob_name = "blob" + source_key = b"01234567890123456789012345678901" # 32 bytes + source_key_b64 = base64.b64encode(source_key).rstrip().decode("ascii") + source_key_hash = hashlib.sha256(source_key).digest() + source_key_hash_b64 = base64.b64encode(source_key_hash).rstrip().decode("ascii") + dest_kms_resource = ( + "projects/test-project-123/" + "locations/us/" + "keyRings/test-ring/" + "cryptoKeys/test-key" + "cryptoKeyVersions/1" + ) + bytes_rewritten = object_size = 42 + api_response = { + "totalBytesRewritten": bytes_rewritten, + "objectSize": object_size, + "done": True, + "resource": {"etag": "DEADBEEF"}, + } + client = mock.Mock(spec=["_post_resource"]) + client._post_resource.return_value = api_response + bucket = _Bucket(client=client) + source = self._make_one(blob_name, bucket=bucket, encryption_key=source_key) + dest = self._make_one(blob_name, bucket=bucket, kms_key_name=dest_kms_resource) + + token, rewritten, size = dest.rewrite(source) + + self.assertIsNone(token) + self.assertEqual(rewritten, bytes_rewritten) + self.assertEqual(size, object_size) + + expected_path = f"/b/name/o/{blob_name}/rewriteTo/b/name/o/{blob_name}" + expected_data = {"kmsKeyName": dest_kms_resource} + # The kmsKeyName version value can't be used in the rewrite request, + # so the client instead ignores it. + expected_query_params = {} + expected_headers = { + "X-Goog-Copy-Source-Encryption-Algorithm": "AES256", + "X-Goog-Copy-Source-Encryption-Key": source_key_b64, + "X-Goog-Copy-Source-Encryption-Key-Sha256": source_key_hash_b64, + } + client._post_resource.assert_called_once_with( + expected_path, + expected_data, + query_params=expected_query_params, + headers=expected_headers, + timeout=self._get_default_timeout(), + retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED, + _target_object=dest, + ) + def test_update_storage_class_invalid(self): blob_name = "blob-name" bucket = _Bucket() From c365d5bbd78292adb6861da3cdfae9ab7b39b844 Mon Sep 17 00:00:00 2001 From: cojenco Date: Fri, 3 Jun 2022 10:11:23 -0700 Subject: [PATCH 23/28] docs: update retry docs (#808) --- docs/storage/retry_timeout.rst | 19 +++++++++++++++++-- google/cloud/storage/retry.py | 3 ++- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/docs/storage/retry_timeout.rst b/docs/storage/retry_timeout.rst index 18e25304a..bc1912658 100644 --- a/docs/storage/retry_timeout.rst +++ b/docs/storage/retry_timeout.rst @@ -73,7 +73,8 @@ for each method, base on its semantics: the same "generation", the library uses its :data:`~google.cloud.storage.retry.DEFAULT_RETRY_IF_GENERATION_SPECIFIED` policy, which retries API requests which returns a "transient" error, - but only if the original request includes an ``ifGenerationMatch`` header. + but only if the original request includes a ``generation`` or + ``ifGenerationMatch`` header. - For API requests which are idempotent only if the bucket or blob has the same "metageneration", the library uses its @@ -99,6 +100,20 @@ explicit policy in your code. bucket = client.get_bucket(BUCKET_NAME, retry=None) +- You can modify the default retry behavior and create a copy of :data:`~google.cloud.storage.retry.DEFAULT_RETRY` + by calling it with a ``with_XXX`` method. E.g.: + +.. code-block:: python + + from google.cloud.storage.retry import DEFAULT_RETRY + + # Customize retry with a deadline of 500 seconds (default=120 seconds). + modified_retry = DEFAULT_RETRY.with_deadline(500.0) + # Customize retry with an initial wait time of 1.5 (default=1.0). + # Customize retry with a wait time multiplier per iteration of 1.2 (default=2.0). + # Customize retry with a maximum wait time of 45.0 (default=60.0). + modified_retry = modified_retry.with_delay(initial=1.5, multiplier=1.2, maximum=45.0) + - You can pass an instance of :class:`google.api_core.retry.Retry` to enable retries; the passed object will define retriable response codes and errors, as well as configuring backoff and retry interval options. E.g.: @@ -140,5 +155,5 @@ explicit policy in your code. my_retry_policy = Retry(predicate=is_retryable) my_cond_policy = ConditionalRetryPolicy( - my_retry_policy, conditional_predicate=is_etag_in_data) + my_retry_policy, conditional_predicate=is_etag_in_data, ["query_params"]) bucket = client.get_bucket(BUCKET_NAME, retry=my_cond_policy) diff --git a/google/cloud/storage/retry.py b/google/cloud/storage/retry.py index fb7a8e4de..a9fb3bb3f 100644 --- a/google/cloud/storage/retry.py +++ b/google/cloud/storage/retry.py @@ -88,7 +88,8 @@ class ConditionalRetryPolicy(object): :type required_kwargs: list(str) :param required_kwargs: A list of keyword argument keys that will be extracted from the API call - and passed into the ``conditional predicate`` in order. + and passed into the ``conditional predicate`` in order. For example, + ``["query_params"]`` is commmonly used for preconditions in query_params. """ def __init__(self, retry_policy, conditional_predicate, required_kwargs): From 1e7cdb655beb2a61a0d1b984c4d0468ec31bf463 Mon Sep 17 00:00:00 2001 From: Jakub Czaplicki Date: Mon, 6 Jun 2022 17:26:10 +0200 Subject: [PATCH 24/28] docs: Update generation_metageneration.rst with a missing space (#798) Co-authored-by: cojenco Co-authored-by: Anthonios Partheniou --- docs/storage/generation_metageneration.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/storage/generation_metageneration.rst b/docs/storage/generation_metageneration.rst index 4a92e534a..eb77dad15 100644 --- a/docs/storage/generation_metageneration.rst +++ b/docs/storage/generation_metageneration.rst @@ -112,7 +112,7 @@ the blob (e.g., makes the operation conditional on whether the blob's current ``generation`` matches the given value. -As a special case, passing ``0`` as the value for``if_generation_match`` +As a special case, passing ``0`` as the value for ``if_generation_match`` makes the operation succeed only if there are no live versions of the blob. From c7b7faef4a649aea866aaac3dde56990f357b9b5 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Tue, 7 Jun 2022 13:09:52 +0200 Subject: [PATCH 25/28] chore(deps): update dependency google-cloud-pubsub to v2.13.0 (#809) --- samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 843c608cd..44c1b701d 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,4 +1,4 @@ -google-cloud-pubsub==2.12.1 +google-cloud-pubsub==2.13.0 google-cloud-storage==2.3.0 pandas===1.3.5; python_version == '3.7' pandas==1.4.2; python_version >= '3.8' From 7b6895c71140d74f25bb6bc1f6d85d7f5d26673f Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Tue, 7 Jun 2022 18:52:59 +0200 Subject: [PATCH 26/28] chore(deps): update dependency backoff to v2.1.0 (#810) --- samples/snippets/requirements-test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements-test.txt b/samples/snippets/requirements-test.txt index 88beb7ba2..ed8c5dde8 100644 --- a/samples/snippets/requirements-test.txt +++ b/samples/snippets/requirements-test.txt @@ -1,3 +1,3 @@ pytest==7.1.2 mock==4.0.3 -backoff==2.0.1 \ No newline at end of file +backoff==2.1.0 \ No newline at end of file From 187cf503194cf636640ca8ba787f9e8c216ea763 Mon Sep 17 00:00:00 2001 From: cojenco Date: Tue, 7 Jun 2022 13:55:26 -0700 Subject: [PATCH 27/28] feat: support OLM Prefix/Suffix (#773) * feat: support OLM Prefix/Suffix * update tests --- google/cloud/storage/bucket.py | 34 ++++++++++++++++++++++++++++++---- tests/system/test_bucket.py | 18 ++++++++++++++++-- tests/unit/test_bucket.py | 22 ++++++++++++++++++++++ 3 files changed, 68 insertions(+), 6 deletions(-) diff --git a/google/cloud/storage/bucket.py b/google/cloud/storage/bucket.py index c98f005c3..1b31baab7 100644 --- a/google/cloud/storage/bucket.py +++ b/google/cloud/storage/bucket.py @@ -163,11 +163,19 @@ class LifecycleRuleConditions(dict): rule action to versioned items with at least one newer version. + :type matches_prefix: list(str) + :param matches_prefix: (Optional) Apply rule action to items which + any prefix matches the beginning of the item name. + :type matches_storage_class: list(str), one or more of :attr:`Bucket.STORAGE_CLASSES`. - :param matches_storage_class: (Optional) Apply rule action to items which + :param matches_storage_class: (Optional) Apply rule action to items whose storage class matches this value. + :type matches_suffix: list(str) + :param matches_suffix: (Optional) Apply rule action to items which + any suffix matches the end of the item name. + :type number_of_newer_versions: int :param number_of_newer_versions: (Optional) Apply rule action to versioned items having N newer versions. @@ -211,6 +219,8 @@ def __init__( custom_time_before=None, days_since_noncurrent_time=None, noncurrent_time_before=None, + matches_prefix=None, + matches_suffix=None, _factory=False, ): conditions = {} @@ -236,15 +246,21 @@ def __init__( if custom_time_before is not None: conditions["customTimeBefore"] = custom_time_before.isoformat() - if not _factory and not conditions: - raise ValueError("Supply at least one condition") - if days_since_noncurrent_time is not None: conditions["daysSinceNoncurrentTime"] = days_since_noncurrent_time if noncurrent_time_before is not None: conditions["noncurrentTimeBefore"] = noncurrent_time_before.isoformat() + if matches_prefix is not None: + conditions["matchesPrefix"] = matches_prefix + + if matches_suffix is not None: + conditions["matchesSuffix"] = matches_suffix + + if not _factory and not conditions: + raise ValueError("Supply at least one condition") + super(LifecycleRuleConditions, self).__init__(conditions) @classmethod @@ -278,11 +294,21 @@ def is_live(self): """Conditon's 'is_live' value.""" return self.get("isLive") + @property + def matches_prefix(self): + """Conditon's 'matches_prefix' value.""" + return self.get("matchesPrefix") + @property def matches_storage_class(self): """Conditon's 'matches_storage_class' value.""" return self.get("matchesStorageClass") + @property + def matches_suffix(self): + """Conditon's 'matches_suffix' value.""" + return self.get("matchesSuffix") + @property def number_of_newer_versions(self): """Conditon's 'number_of_newer_versions' value.""" diff --git a/tests/system/test_bucket.py b/tests/system/test_bucket.py index d8796f5b3..062cc8998 100644 --- a/tests/system/test_bucket.py +++ b/tests/system/test_bucket.py @@ -47,6 +47,8 @@ def test_bucket_lifecycle_rules(storage_client, buckets_to_delete): bucket_name = _helpers.unique_name("w-lifcycle-rules") custom_time_before = datetime.date(2018, 8, 1) noncurrent_before = datetime.date(2018, 8, 1) + matches_prefix = ["storage-sys-test", "gcs-sys-test"] + matches_suffix = ["suffix-test"] with pytest.raises(exceptions.NotFound): storage_client.get_bucket(bucket_name) @@ -59,6 +61,8 @@ def test_bucket_lifecycle_rules(storage_client, buckets_to_delete): custom_time_before=custom_time_before, days_since_noncurrent_time=2, noncurrent_time_before=noncurrent_before, + matches_prefix=matches_prefix, + matches_suffix=matches_suffix, ) bucket.add_lifecycle_set_storage_class_rule( constants.COLDLINE_STORAGE_CLASS, @@ -77,6 +81,8 @@ def test_bucket_lifecycle_rules(storage_client, buckets_to_delete): custom_time_before=custom_time_before, days_since_noncurrent_time=2, noncurrent_time_before=noncurrent_before, + matches_prefix=matches_prefix, + matches_suffix=matches_suffix, ), LifecycleRuleSetStorageClass( constants.COLDLINE_STORAGE_CLASS, @@ -95,9 +101,17 @@ def test_bucket_lifecycle_rules(storage_client, buckets_to_delete): assert list(bucket.lifecycle_rules) == expected_rules # Test modifying lifecycle rules - expected_rules[0] = LifecycleRuleDelete(age=30) + expected_rules[0] = LifecycleRuleDelete( + age=30, + matches_prefix=["new-prefix"], + matches_suffix=["new-suffix"], + ) rules = list(bucket.lifecycle_rules) - rules[0]["condition"] = {"age": 30} + rules[0]["condition"] = { + "age": 30, + "matchesPrefix": ["new-prefix"], + "matchesSuffix": ["new-suffix"], + } bucket.lifecycle_rules = rules bucket.patch() diff --git a/tests/unit/test_bucket.py b/tests/unit/test_bucket.py index f253db3e1..d5206f287 100644 --- a/tests/unit/test_bucket.py +++ b/tests/unit/test_bucket.py @@ -230,6 +230,28 @@ def test_ctor_w_noncurrent_time_before(self): self.assertEqual(conditions.number_of_newer_versions, 3) self.assertEqual(conditions.noncurrent_time_before, noncurrent_before) + def test_ctor_w_matches_prefix(self): + conditions = self._make_one(matches_prefix=["test-prefix"]) + expected = {"matchesPrefix": ["test-prefix"]} + self.assertEqual(dict(conditions), expected) + self.assertIsNone(conditions.age) + self.assertIsNone(conditions.created_before) + self.assertIsNone(conditions.is_live) + self.assertIsNone(conditions.matches_storage_class) + self.assertIsNone(conditions.matches_suffix) + self.assertEqual(conditions.matches_prefix, ["test-prefix"]) + + def test_ctor_w_matches_suffix(self): + conditions = self._make_one(matches_suffix=["test-suffix"]) + expected = {"matchesSuffix": ["test-suffix"]} + self.assertEqual(dict(conditions), expected) + self.assertIsNone(conditions.age) + self.assertIsNone(conditions.created_before) + self.assertIsNone(conditions.is_live) + self.assertIsNone(conditions.matches_storage_class) + self.assertIsNone(conditions.matches_prefix) + self.assertEqual(conditions.matches_suffix, ["test-suffix"]) + def test_from_api_repr(self): import datetime From a62cbb2d8774c079e15dad3161f3d48297f67686 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Wed, 8 Jun 2022 11:07:58 -0700 Subject: [PATCH 28/28] chore(main): release 2.4.0 (#768) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- CHANGELOG.md | 21 +++++++++++++++++++++ google/cloud/storage/version.py | 2 +- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 58fb809de..6bc2a1ea9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,27 @@ [1]: https://pypi.org/project/google-cloud-storage/#history +## [2.4.0](https://github.com/googleapis/python-storage/compare/v2.3.0...v2.4.0) (2022-06-07) + + +### Features + +* add AbortIncompleteMultipartUpload lifecycle rule ([#765](https://github.com/googleapis/python-storage/issues/765)) ([b2e5150](https://github.com/googleapis/python-storage/commit/b2e5150f191c04acb47ad98cef88512451aff81d)) +* support OLM Prefix/Suffix ([#773](https://github.com/googleapis/python-storage/issues/773)) ([187cf50](https://github.com/googleapis/python-storage/commit/187cf503194cf636640ca8ba787f9e8c216ea763)) + + +### Bug Fixes + +* fix rewrite object in CMEK enabled bucket ([#807](https://github.com/googleapis/python-storage/issues/807)) ([9b3cbf3](https://github.com/googleapis/python-storage/commit/9b3cbf3789c21462eac3c776cd29df12701e792f)) + + +### Documentation + +* fix changelog header to consistent size ([#802](https://github.com/googleapis/python-storage/issues/802)) ([4dd0907](https://github.com/googleapis/python-storage/commit/4dd0907b68e20d1ffcd0fe350831867197917e0d)) +* **samples:** Update the Recovery Point Objective (RPO) sample output ([#725](https://github.com/googleapis/python-storage/issues/725)) ([b0bf411](https://github.com/googleapis/python-storage/commit/b0bf411f8fec8712b3eeb99a2dd33de6d82312f8)) +* Update generation_metageneration.rst with a missing space ([#798](https://github.com/googleapis/python-storage/issues/798)) ([1e7cdb6](https://github.com/googleapis/python-storage/commit/1e7cdb655beb2a61a0d1b984c4d0468ec31bf463)) +* update retry docs ([#808](https://github.com/googleapis/python-storage/issues/808)) ([c365d5b](https://github.com/googleapis/python-storage/commit/c365d5bbd78292adb6861da3cdfae9ab7b39b844)) + ## [2.3.0](https://github.com/googleapis/python-storage/compare/v2.2.1...v2.3.0) (2022-04-12) diff --git a/google/cloud/storage/version.py b/google/cloud/storage/version.py index 999199f5a..fe11624d9 100644 --- a/google/cloud/storage/version.py +++ b/google/cloud/storage/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.3.0" +__version__ = "2.4.0"