From 7f771077511d6b446686724f48514d5b903ec036 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Thu, 13 Jan 2022 20:25:16 +0100 Subject: [PATCH 1/9] chore(deps): update dependency google-cloud-storage to v2 (#690) --- samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 14c2e74b4..443e0a601 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,4 +1,4 @@ google-cloud-pubsub==2.9.0 -google-cloud-storage==1.44.0 +google-cloud-storage==2.0.0 pandas==1.3.5; python_version > '3.6' pandas==1.1.5; python_version < '3.7' From b53feaaaf0af92ea9e1d16abd317f798f2b6e17a Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Thu, 13 Jan 2022 20:38:16 +0000 Subject: [PATCH 2/9] build: switch to release-please for tagging (#691) --- .github/.OwlBot.lock.yaml | 2 +- .github/release-please.yml | 1 + .github/release-trigger.yml | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 .github/release-trigger.yml diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 6b8a73b31..ff5126c18 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -1,3 +1,3 @@ docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:36a95b8f494e4674dc9eee9af98961293b51b86b3649942aac800ae6c1f796d4 + digest: sha256:dfa9b663b32de8b5b327e32c1da665a80de48876558dd58091d8160c60ad7355 diff --git a/.github/release-please.yml b/.github/release-please.yml index 4507ad059..466597e5b 100644 --- a/.github/release-please.yml +++ b/.github/release-please.yml @@ -1 +1,2 @@ releaseType: python +handleGHRelease: true diff --git a/.github/release-trigger.yml b/.github/release-trigger.yml new file mode 100644 index 000000000..d4ca94189 --- /dev/null +++ b/.github/release-trigger.yml @@ -0,0 +1 @@ +enabled: true From fe1855a5f23dd49f5d2f830d9ae39b8f2f0a4aaf Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Fri, 14 Jan 2022 09:09:15 -0500 Subject: [PATCH 3/9] chore(python): update release.sh to use keystore (#692) Source-Link: https://github.com/googleapis/synthtool/commit/69fda12e2994f0b595a397e8bb6e3e9f380524eb Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:ae600f36b6bc972b368367b6f83a1d91ec2c82a4a116b383d67d547c56fe6de3 Co-authored-by: Owl Bot --- .github/.OwlBot.lock.yaml | 2 +- .kokoro/release.sh | 2 +- .kokoro/release/common.cfg | 12 +++++++++++- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index ff5126c18..eecb84c21 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -1,3 +1,3 @@ docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:dfa9b663b32de8b5b327e32c1da665a80de48876558dd58091d8160c60ad7355 + digest: sha256:ae600f36b6bc972b368367b6f83a1d91ec2c82a4a116b383d67d547c56fe6de3 diff --git a/.kokoro/release.sh b/.kokoro/release.sh index 7970969eb..64a3c6ab3 100755 --- a/.kokoro/release.sh +++ b/.kokoro/release.sh @@ -26,7 +26,7 @@ python3 -m pip install --upgrade twine wheel setuptools export PYTHONUNBUFFERED=1 # Move into the package, build the distribution and upload. -TWINE_PASSWORD=$(cat "${KOKORO_GFILE_DIR}/secret_manager/google-cloud-pypi-token") +TWINE_PASSWORD=$(cat "${KOKORO_KEYSTORE_DIR}/73713_google-cloud-pypi-token-keystore-1") cd github/python-storage python3 setup.py sdist bdist_wheel twine upload --username __token__ --password "${TWINE_PASSWORD}" dist/* diff --git a/.kokoro/release/common.cfg b/.kokoro/release/common.cfg index 58a3ff6b8..b83a57783 100644 --- a/.kokoro/release/common.cfg +++ b/.kokoro/release/common.cfg @@ -23,8 +23,18 @@ env_vars: { value: "github/python-storage/.kokoro/release.sh" } +# Fetch PyPI password +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73713 + keyname: "google-cloud-pypi-token-keystore-1" + } + } +} + # Tokens needed to report release status back to GitHub env_vars: { key: "SECRET_MANAGER_KEYS" - value: "releasetool-publish-reporter-app,releasetool-publish-reporter-googleapis-installation,releasetool-publish-reporter-pem,google-cloud-pypi-token" + value: "releasetool-publish-reporter-app,releasetool-publish-reporter-googleapis-installation,releasetool-publish-reporter-pem" } From 8aa4130ee068a1922161c8ca54a53a4a51d65ce0 Mon Sep 17 00:00:00 2001 From: Aaron Gabriel Neyer Date: Tue, 18 Jan 2022 11:47:49 -0700 Subject: [PATCH 4/9] feat: remove python 3.6 support (#689) * remove python 3.6 support * few more things * try deleting kokoro python3.6 samples --- .kokoro/samples/python3.6/common.cfg | 40 --------------------- .kokoro/samples/python3.6/continuous.cfg | 7 ---- .kokoro/samples/python3.6/periodic-head.cfg | 11 ------ .kokoro/samples/python3.6/periodic.cfg | 6 ---- .kokoro/samples/python3.6/presubmit.cfg | 6 ---- README.rst | 9 +++-- noxfile.py | 2 +- owlbot.py | 2 +- samples/snippets/noxfile_config.py | 2 +- setup.py | 3 +- testing/constraints-3.6.txt | 12 ------- 11 files changed, 10 insertions(+), 90 deletions(-) delete mode 100644 .kokoro/samples/python3.6/common.cfg delete mode 100644 .kokoro/samples/python3.6/continuous.cfg delete mode 100644 .kokoro/samples/python3.6/periodic-head.cfg delete mode 100644 .kokoro/samples/python3.6/periodic.cfg delete mode 100644 .kokoro/samples/python3.6/presubmit.cfg delete mode 100644 testing/constraints-3.6.txt diff --git a/.kokoro/samples/python3.6/common.cfg b/.kokoro/samples/python3.6/common.cfg deleted file mode 100644 index 985a0cbfb..000000000 --- a/.kokoro/samples/python3.6/common.cfg +++ /dev/null @@ -1,40 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -# Build logs will be here -action { - define_artifacts { - regex: "**/*sponge_log.xml" - } -} - -# Specify which tests to run -env_vars: { - key: "RUN_TESTS_SESSION" - value: "py-3.6" -} - -# Declare build specific Cloud project. -env_vars: { - key: "BUILD_SPECIFIC_GCLOUD_PROJECT" - value: "python-docs-samples-tests-py36" -} - -env_vars: { - key: "TRAMPOLINE_BUILD_FILE" - value: "github/python-storage/.kokoro/test-samples.sh" -} - -# Configure the docker image for kokoro-trampoline. -env_vars: { - key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-kokoro-resources/python-samples-testing-docker" -} - -# Download secrets for samples -gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" - -# Download trampoline resources. -gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" - -# Use the trampoline script to run in docker. -build_file: "python-storage/.kokoro/trampoline_v2.sh" \ No newline at end of file diff --git a/.kokoro/samples/python3.6/continuous.cfg b/.kokoro/samples/python3.6/continuous.cfg deleted file mode 100644 index 7218af149..000000000 --- a/.kokoro/samples/python3.6/continuous.cfg +++ /dev/null @@ -1,7 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -env_vars: { - key: "INSTALL_LIBRARY_FROM_SOURCE" - value: "True" -} - diff --git a/.kokoro/samples/python3.6/periodic-head.cfg b/.kokoro/samples/python3.6/periodic-head.cfg deleted file mode 100644 index 5d0faf58f..000000000 --- a/.kokoro/samples/python3.6/periodic-head.cfg +++ /dev/null @@ -1,11 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -env_vars: { - key: "INSTALL_LIBRARY_FROM_SOURCE" - value: "True" -} - -env_vars: { - key: "TRAMPOLINE_BUILD_FILE" - value: "github/python-storage/.kokoro/test-samples-against-head.sh" -} diff --git a/.kokoro/samples/python3.6/periodic.cfg b/.kokoro/samples/python3.6/periodic.cfg deleted file mode 100644 index 71cd1e597..000000000 --- a/.kokoro/samples/python3.6/periodic.cfg +++ /dev/null @@ -1,6 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -env_vars: { - key: "INSTALL_LIBRARY_FROM_SOURCE" - value: "False" -} diff --git a/.kokoro/samples/python3.6/presubmit.cfg b/.kokoro/samples/python3.6/presubmit.cfg deleted file mode 100644 index a1c8d9759..000000000 --- a/.kokoro/samples/python3.6/presubmit.cfg +++ /dev/null @@ -1,6 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -env_vars: { - key: "INSTALL_LIBRARY_FROM_SOURCE" - value: "True" -} \ No newline at end of file diff --git a/README.rst b/README.rst index 5419ae509..8a1304b73 100644 --- a/README.rst +++ b/README.rst @@ -43,7 +43,7 @@ Installation `venv`_ is a tool to create isolated Python environments. The basic problem it addresses is one of dependencies and versions, and indirectly permissions. -Make sure you're using Python 3.3 or later, which includes `venv`_ by default. +Make sure you're using Python 3.7 or later, which includes `venv`_ by default. With `venv`, it's possible to install this library without needing system install permissions, and without clashing with the installed system dependencies. @@ -54,14 +54,17 @@ dependencies. Supported Python Versions ^^^^^^^^^^^^^^^^^^^^^^^^^ -Python >= 3.6 +Python >= 3.7 Deprecated Python Versions ^^^^^^^^^^^^^^^^^^^^^^^^^^ -Python == 2.7: Python 2.7 support will be removed sometime after January 1, 2020. Unsupported Python Versions ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Python == 3.6: the last released version which supported Python 3.6 was +``google-cloud-storage 2.0.0``, released 2022-01-12. + Python == 3.5: the last released version which supported Python 3.5 was ``google-cloud-storage 1.32.0``, released 2020-10-16. diff --git a/noxfile.py b/noxfile.py index 318bc3957..069a486c1 100644 --- a/noxfile.py +++ b/noxfile.py @@ -29,7 +29,7 @@ DEFAULT_PYTHON_VERSION = "3.8" SYSTEM_TEST_PYTHON_VERSIONS = ["3.8"] -UNIT_TEST_PYTHON_VERSIONS = ["3.6", "3.7", "3.8", "3.9", "3.10"] +UNIT_TEST_PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10"] CONFORMANCE_TEST_PYTHON_VERSIONS = ["3.8"] _DEFAULT_STORAGE_HOST = "https://storage.googleapis.com" diff --git a/owlbot.py b/owlbot.py index 828536f24..c4d3b0ccb 100644 --- a/owlbot.py +++ b/owlbot.py @@ -26,7 +26,7 @@ templated_files = common.py_library( cov_level=100, split_system_tests=True, - unit_test_python_versions=["3.6", "3.7", "3.8", "3.9", "3.10"], + unit_test_python_versions=["3.7", "3.8", "3.9", "3.10"], system_test_external_dependencies=[ "google-cloud-iam", "google-cloud-pubsub < 2.0.0", diff --git a/samples/snippets/noxfile_config.py b/samples/snippets/noxfile_config.py index 463da97de..ecd7fdce7 100644 --- a/samples/snippets/noxfile_config.py +++ b/samples/snippets/noxfile_config.py @@ -72,7 +72,7 @@ def get_cloud_kms_key(): TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - 'ignored_versions': ["2.7"], + 'ignored_versions': ["2.7", "3.6"], # An envvar key for determining the project id to use. Change it # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a diff --git a/setup.py b/setup.py index 9264e4f56..8dae147d9 100644 --- a/setup.py +++ b/setup.py @@ -78,7 +78,6 @@ "License :: OSI Approved :: Apache Software License", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", @@ -91,7 +90,7 @@ namespace_packages=namespaces, install_requires=dependencies, extras_require=extras, - python_requires=">=3.6", + python_requires=">=3.7", include_package_data=True, zip_safe=False, ) diff --git a/testing/constraints-3.6.txt b/testing/constraints-3.6.txt deleted file mode 100644 index a2729fd6f..000000000 --- a/testing/constraints-3.6.txt +++ /dev/null @@ -1,12 +0,0 @@ -# This constraints file is used to check that lower bounds -# are correct in setup.py -# List *all* library dependencies and extras in this file. -# Pin the version to the lower bound. -# -# e.g., if setup.py has "foo >= 1.14.0, < 2.0.0dev", -# Then this file should have foo==1.14.0 -google-auth==1.25.0 -google-api-core==1.29.0 -google-cloud-core==1.6.0 -google-resumable-media==1.3.0 -requests==2.18.0 From 4dafc815470480ce9de7f0357e331d3fbd0ae9b7 Mon Sep 17 00:00:00 2001 From: Aaron Gabriel Neyer Date: Tue, 18 Jan 2022 14:12:01 -0700 Subject: [PATCH 5/9] feat: add turbo replication support and samples (#622) * feat: add turbo replication support * lintfix * nother-lint-fix * hmm weird * i need to learn how to use lint better... * . * . * how about now? * take rpo out of constructor * add unit tests * add link to docs * ensure inclusion of "rpo" in bucket._changes * add rpo samples * lint it * address cathys nits * fix a little test thing * start to fix weirdness, creating issue to address more fully * change it back --- google/cloud/storage/bucket.py | 23 +++++++ google/cloud/storage/constants.py | 12 ++++ samples/snippets/rpo_test.py | 61 +++++++++++++++++++ ...storage_create_bucket_turbo_replication.py | 48 +++++++++++++++ samples/snippets/storage_get_rpo.py | 48 +++++++++++++++ .../snippets/storage_set_rpo_async_turbo.py | 48 +++++++++++++++ samples/snippets/storage_set_rpo_default.py | 48 +++++++++++++++ tests/system/test_bucket.py | 19 ++++++ tests/unit/test_bucket.py | 10 +++ 9 files changed, 317 insertions(+) create mode 100644 samples/snippets/rpo_test.py create mode 100644 samples/snippets/storage_create_bucket_turbo_replication.py create mode 100644 samples/snippets/storage_get_rpo.py create mode 100644 samples/snippets/storage_set_rpo_async_turbo.py create mode 100644 samples/snippets/storage_set_rpo_default.py diff --git a/google/cloud/storage/bucket.py b/google/cloud/storage/bucket.py index 6f738976b..d071615ef 100644 --- a/google/cloud/storage/bucket.py +++ b/google/cloud/storage/bucket.py @@ -631,6 +631,29 @@ def _set_properties(self, value): self._label_removals.clear() return super(Bucket, self)._set_properties(value) + @property + def rpo(self): + """Get the RPO (Recovery Point Objective) of this bucket + + See: https://cloud.google.com/storage/docs/managing-turbo-replication + + "ASYNC_TURBO" or "DEFAULT" + :rtype: str + """ + return self._properties.get("rpo") + + @rpo.setter + def rpo(self, value): + """ + Set the RPO (Recovery Point Objective) of this bucket. + + See: https://cloud.google.com/storage/docs/managing-turbo-replication + + :type value: str + :param value: "ASYNC_TURBO" or "DEFAULT" + """ + self._patch_property("rpo", value) + @property def user_project(self): """Project ID to be billed for API requests made via this bucket. diff --git a/google/cloud/storage/constants.py b/google/cloud/storage/constants.py index 2e1c1dd2a..132f4e40a 100644 --- a/google/cloud/storage/constants.py +++ b/google/cloud/storage/constants.py @@ -117,3 +117,15 @@ See: https://cloud.google.com/storage/docs/public-access-prevention """ + +RPO_ASYNC_TURBO = "ASYNC_TURBO" +"""Turbo Replication RPO + +See: https://cloud.google.com/storage/docs/managing-turbo-replication +""" + +RPO_DEFAULT = "DEFAULT" +"""Default RPO + +See: https://cloud.google.com/storage/docs/managing-turbo-replication +""" diff --git a/samples/snippets/rpo_test.py b/samples/snippets/rpo_test.py new file mode 100644 index 000000000..d084710a9 --- /dev/null +++ b/samples/snippets/rpo_test.py @@ -0,0 +1,61 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import uuid + +from google.cloud import storage +import pytest + +import storage_create_bucket_turbo_replication +import storage_get_rpo +import storage_set_rpo_async_turbo +import storage_set_rpo_default + + +@pytest.fixture +def dual_region_bucket(): + """Yields a dual region bucket that is deleted after the test completes.""" + bucket = None + while bucket is None or bucket.exists(): + bucket_name = "bucket-lock-{}".format(uuid.uuid4()) + bucket = storage.Client().bucket(bucket_name) + bucket.location = "NAM4" + bucket.create() + yield bucket + bucket.delete(force=True) + + +def test_get_rpo(dual_region_bucket, capsys): + storage_get_rpo.get_rpo(dual_region_bucket.name) + out, _ = capsys.readouterr() + assert f"RPO for {dual_region_bucket.name} is DEFAULT." in out + + +def test_set_rpo_async_turbo(dual_region_bucket, capsys): + storage_set_rpo_async_turbo.set_rpo_async_turbo(dual_region_bucket.name) + out, _ = capsys.readouterr() + assert f"RPO is ASYNC_TURBO for {dual_region_bucket.name}." in out + + +def test_set_rpo_default(dual_region_bucket, capsys): + storage_set_rpo_default.set_rpo_default(dual_region_bucket.name) + out, _ = capsys.readouterr() + assert f"RPO is DEFAULT for {dual_region_bucket.name}." in out + + +def test_create_bucket_turbo_replication(capsys): + bucket_name = "test-rpo-{}".format(uuid.uuid4()) + storage_create_bucket_turbo_replication.create_bucket_turbo_replication(bucket_name) + out, _ = capsys.readouterr() + assert f"{bucket_name} created with RPO ASYNC_TURBO in NAM4." in out diff --git a/samples/snippets/storage_create_bucket_turbo_replication.py b/samples/snippets/storage_create_bucket_turbo_replication.py new file mode 100644 index 000000000..68f0ba482 --- /dev/null +++ b/samples/snippets/storage_create_bucket_turbo_replication.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python + +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +"""Sample that creates a new bucket with dual-region and turbo replication. +This sample is used on this page: + https://cloud.google.com/storage/docs/managing-turbo-replication +For more information, see README.md. +""" + +# [START storage_create_bucket_turbo_replication] + +from google.cloud import storage +from google.cloud.storage.constants import RPO_ASYNC_TURBO + + +def create_bucket_turbo_replication(bucket_name): + """Creates dual-region bucket with turbo replication enabled.""" + # The ID of your GCS bucket + # bucket_name = "my-bucket" + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + bucket.location = "NAM4" + bucket.rpo = RPO_ASYNC_TURBO + bucket.create() + + print(f"{bucket.name} created with RPO {bucket.rpo} in {bucket.location}.") + + +# [END storage_create_bucket_turbo_replication] + +if __name__ == "__main__": + create_bucket_turbo_replication(bucket_name=sys.argv[1]) diff --git a/samples/snippets/storage_get_rpo.py b/samples/snippets/storage_get_rpo.py new file mode 100644 index 000000000..29ae186fa --- /dev/null +++ b/samples/snippets/storage_get_rpo.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python + +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +"""Sample that gets RPO (Recovery Point Objective) of a bucket +This sample is used on this page: + https://cloud.google.com/storage/docs/managing-turbo-replication +For more information, see README.md. +""" + +# [START storage_get_rpo] + +from google.cloud import storage +from google.cloud.storage.constants import RPO_DEFAULT + + +def get_rpo(bucket_name): + """Gets the RPO of the bucket""" + # The ID of your GCS bucket + # bucket_name = "my-bucket" + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + + bucket.rpo = RPO_DEFAULT + rpo = bucket.rpo + + print(f"RPO for {bucket.name} is {rpo}.") + + +# [END storage_get_rpo] + +if __name__ == "__main__": + get_rpo(bucket_name=sys.argv[1]) diff --git a/samples/snippets/storage_set_rpo_async_turbo.py b/samples/snippets/storage_set_rpo_async_turbo.py new file mode 100644 index 000000000..10b4c67a3 --- /dev/null +++ b/samples/snippets/storage_set_rpo_async_turbo.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python + +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +"""Sample that sets RPO (Recovery Point Objective) to ASYNC_TURBO +This sample is used on this page: + https://cloud.google.com/storage/docs/managing-turbo-replication +For more information, see README.md. +""" + +# [START storage_set_rpo_async_turbo] + +from google.cloud import storage +from google.cloud.storage.constants import RPO_ASYNC_TURBO + + +def set_rpo_async_turbo(bucket_name): + """Sets the RPO to ASYNC_TURBO, enabling the turbo replication feature""" + # The ID of your GCS bucket + # bucket_name = "my-bucket" + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + + bucket.rpo = RPO_ASYNC_TURBO + bucket.patch() + + print(f"RPO is ASYNC_TURBO for {bucket.name}.") + + +# [END storage_set_rpo_async_turbo] + +if __name__ == "__main__": + set_rpo_async_turbo(bucket_name=sys.argv[1]) diff --git a/samples/snippets/storage_set_rpo_default.py b/samples/snippets/storage_set_rpo_default.py new file mode 100644 index 000000000..8d41b1fe0 --- /dev/null +++ b/samples/snippets/storage_set_rpo_default.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python + +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +"""Sample that sets RPO (Recovery Point Objective) to default +This sample is used on this page: + https://cloud.google.com/storage/docs/managing-turbo-replication +For more information, see README.md. +""" + +# [START storage_set_rpo_default] + +from google.cloud import storage +from google.cloud.storage.constants import RPO_DEFAULT + + +def set_rpo_default(bucket_name): + """Sets the RPO to DEFAULT, disabling the turbo replication feature""" + # The ID of your GCS bucket + # bucket_name = "my-bucket" + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + + bucket.rpo = RPO_DEFAULT + bucket.patch() + + print(f"RPO is DEFAULT for {bucket.name}.") + + +# [END storage_set_rpo_default] + +if __name__ == "__main__": + set_rpo_default(bucket_name=sys.argv[1]) diff --git a/tests/system/test_bucket.py b/tests/system/test_bucket.py index 78fa135ff..dc1869d2f 100644 --- a/tests/system/test_bucket.py +++ b/tests/system/test_bucket.py @@ -885,3 +885,22 @@ def test_new_bucket_created_w_enforced_pap( constants.PUBLIC_ACCESS_PREVENTION_INHERITED, ] assert not bucket.iam_configuration.uniform_bucket_level_access_enabled + + +def test_new_bucket_with_rpo( + storage_client, buckets_to_delete, blobs_to_delete, +): + from google.cloud.storage import constants + + bucket_name = _helpers.unique_name("new-w-turbo-replication") + bucket = storage_client.create_bucket(bucket_name, location="NAM4") + buckets_to_delete.append(bucket) + + assert bucket.rpo == constants.RPO_DEFAULT + + bucket.rpo = constants.RPO_ASYNC_TURBO + bucket.patch() + + bucket_from_server = storage_client.get_bucket(bucket_name) + + assert bucket_from_server.rpo == constants.RPO_ASYNC_TURBO diff --git a/tests/unit/test_bucket.py b/tests/unit/test_bucket.py index 8bccee19c..122233b6e 100644 --- a/tests/unit/test_bucket.py +++ b/tests/unit/test_bucket.py @@ -25,6 +25,8 @@ from google.cloud.storage.constants import PUBLIC_ACCESS_PREVENTION_ENFORCED from google.cloud.storage.constants import PUBLIC_ACCESS_PREVENTION_INHERITED from google.cloud.storage.constants import PUBLIC_ACCESS_PREVENTION_UNSPECIFIED +from google.cloud.storage.constants import RPO_DEFAULT +from google.cloud.storage.constants import RPO_ASYNC_TURBO def _create_signing_credentials(): @@ -2476,6 +2478,14 @@ def test_location_type_getter_set(self): bucket = self._make_one(properties=properties) self.assertEqual(bucket.location_type, REGION_LOCATION_TYPE) + def test_rpo_getter_and_setter(self): + bucket = self._make_one() + bucket.rpo = RPO_ASYNC_TURBO + self.assertEqual(bucket.rpo, RPO_ASYNC_TURBO) + bucket.rpo = RPO_DEFAULT + self.assertIn("rpo", bucket._changes) + self.assertEqual(bucket.rpo, RPO_DEFAULT) + def test_get_logging_w_prefix(self): NAME = "name" LOG_BUCKET = "logs" From 71a453531603b442e26f1c78ab519cc0248e16c8 Mon Sep 17 00:00:00 2001 From: Aaron Gabriel Neyer Date: Tue, 18 Jan 2022 14:53:05 -0700 Subject: [PATCH 6/9] samples: add async upload sample (#665) * samples: add async upload sample * make python3.6 compatible * woops, this one too * so annoying, can we deprecate 3.6 too? * be gone with you 3.6! * add comment clarifying how sample is run --- samples/snippets/snippets_test.py | 8 ++++ samples/snippets/storage_async_upload.py | 61 ++++++++++++++++++++++++ 2 files changed, 69 insertions(+) create mode 100644 samples/snippets/storage_async_upload.py diff --git a/samples/snippets/snippets_test.py b/samples/snippets/snippets_test.py index 7c0a5b91d..28b35340b 100644 --- a/samples/snippets/snippets_test.py +++ b/samples/snippets/snippets_test.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import asyncio import os import tempfile import time @@ -23,6 +24,7 @@ import requests import storage_add_bucket_label +import storage_async_upload import storage_batch_request import storage_bucket_delete_default_kms_key import storage_change_default_storage_class @@ -213,6 +215,12 @@ def test_upload_blob_with_kms(test_bucket): assert kms_blob.kms_key_name.startswith(KMS_KEY) +def test_async_upload(bucket, capsys): + asyncio.run(storage_async_upload.async_upload_blob(bucket.name)) + out, _ = capsys.readouterr() + assert f"Uploaded 3 files to bucket {bucket.name}" in out + + def test_download_byte_range(test_blob): with tempfile.NamedTemporaryFile() as dest_file: storage_download_byte_range.download_byte_range( diff --git a/samples/snippets/storage_async_upload.py b/samples/snippets/storage_async_upload.py new file mode 100644 index 000000000..25aabb63e --- /dev/null +++ b/samples/snippets/storage_async_upload.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python + +# Copyright 2021 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +import sys + + +"""Sample that asynchronously uploads a file to GCS +""" + + +# [START storage_async_upload] +# This sample can be run by calling `async.run(async_upload_blob('bucket_name'))` +async def async_upload_blob(bucket_name): + """Uploads a number of files in parallel to the bucket.""" + # The ID of your GCS bucket + # bucket_name = "your-bucket-name" + import asyncio + from functools import partial + from google.cloud import storage + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + + loop = asyncio.get_running_loop() + + tasks = [] + count = 3 + for x in range(count): + blob_name = f"async_sample_blob_{x}" + content = f"Hello world #{x}" + blob = bucket.blob(blob_name) + # The first arg, None, tells it to use the default loops executor + tasks.append(loop.run_in_executor(None, partial(blob.upload_from_string, content))) + + # If the method returns a value (such as download_as_string), gather will return the values + await asyncio.gather(*tasks) + + print(f"Uploaded {count} files to bucket {bucket_name}") + + +# [END storage_async_upload] + + +if __name__ == "__main__": + asyncio.run(async_upload_blob( + bucket_name=sys.argv[1] + )) From 8789afaaa1b2bd6f03fae72e3d87ce004ec10129 Mon Sep 17 00:00:00 2001 From: cojenco Date: Tue, 18 Jan 2022 15:31:50 -0800 Subject: [PATCH 7/9] feat: avoid authentication with storage emulator (#679) * feat: avoid authentication with storage emulator * add unit tests * add more unit tests * add support for corner case project set in envvar * add _helpers unit test * allow flexibility for locational endpoints --- google/cloud/storage/_helpers.py | 7 ++++ google/cloud/storage/client.py | 29 ++++++++++++---- tests/unit/test__helpers.py | 28 +++++++++++++++ tests/unit/test_client.py | 59 ++++++++++++++++++++++++++++++++ 4 files changed, 116 insertions(+), 7 deletions(-) diff --git a/google/cloud/storage/_helpers.py b/google/cloud/storage/_helpers.py index c8359dc1b..c3b104edd 100644 --- a/google/cloud/storage/_helpers.py +++ b/google/cloud/storage/_helpers.py @@ -23,6 +23,7 @@ from urllib.parse import urlsplit from google import resumable_media +from google.auth import environment_vars from google.cloud.storage.constants import _DEFAULT_TIMEOUT from google.cloud.storage.retry import DEFAULT_RETRY from google.cloud.storage.retry import DEFAULT_RETRY_IF_METAGENERATION_SPECIFIED @@ -62,6 +63,12 @@ def _get_storage_host(): return os.environ.get(STORAGE_EMULATOR_ENV_VAR, _DEFAULT_STORAGE_HOST) +def _get_environ_project(): + return os.getenv( + environment_vars.PROJECT, os.getenv(environment_vars.LEGACY_PROJECT), + ) + + def _validate_name(name): """Pre-flight ``Bucket`` name validation. diff --git a/google/cloud/storage/client.py b/google/cloud/storage/client.py index 9d1d49af8..042b3513e 100644 --- a/google/cloud/storage/client.py +++ b/google/cloud/storage/client.py @@ -31,6 +31,7 @@ from google.cloud._helpers import _LocalStack, _NOW from google.cloud.client import ClientWithProject from google.cloud.exceptions import NotFound +from google.cloud.storage._helpers import _get_environ_project from google.cloud.storage._helpers import _get_storage_host from google.cloud.storage._helpers import _DEFAULT_STORAGE_HOST from google.cloud.storage._helpers import _bucket_bound_hostname_url @@ -121,13 +122,6 @@ def __init__( if project is _marker: project = None - super(Client, self).__init__( - project=project, - credentials=credentials, - client_options=client_options, - _http=_http, - ) - kw_args = {"client_info": client_info} # `api_endpoint` should be only set by the user via `client_options`, @@ -148,6 +142,27 @@ def __init__( api_endpoint = client_options.api_endpoint kw_args["api_endpoint"] = api_endpoint + # Use anonymous credentials and no project when + # STORAGE_EMULATOR_HOST or a non-default api_endpoint is set. + if ( + kw_args["api_endpoint"] is not None + and kw_args["api_endpoint"].find("storage.googleapis.com") < 0 + ): + if credentials is None: + credentials = AnonymousCredentials() + if project is None: + project = _get_environ_project() + if project is None: + no_project = True + project = "" + + super(Client, self).__init__( + project=project, + credentials=credentials, + client_options=client_options, + _http=_http, + ) + if no_project: self.project = None diff --git a/tests/unit/test__helpers.py b/tests/unit/test__helpers.py index b99b78cfd..1b0a033dc 100644 --- a/tests/unit/test__helpers.py +++ b/tests/unit/test__helpers.py @@ -46,6 +46,34 @@ def test_w_env_var(self): self.assertEqual(host, HOST) +class Test__get_environ_project(unittest.TestCase): + @staticmethod + def _call_fut(): + from google.cloud.storage._helpers import _get_environ_project + + return _get_environ_project() + + def test_wo_env_var(self): + with mock.patch("os.environ", {}): + project = self._call_fut() + + self.assertEqual(project, None) + + def test_w_env_var(self): + from google.auth import environment_vars + + PROJECT = "environ-project" + + with mock.patch("os.environ", {environment_vars.PROJECT: PROJECT}): + project = self._call_fut() + self.assertEqual(project, PROJECT) + + with mock.patch("os.environ", {environment_vars.LEGACY_PROJECT: PROJECT}): + project = self._call_fut() + + self.assertEqual(project, PROJECT) + + class Test_PropertyMixin(unittest.TestCase): @staticmethod def _get_default_timeout(): diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index c7abf5b0d..2f76041bd 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -236,6 +236,65 @@ def test_ctor_mtls(self): self.assertEqual(client._connection.ALLOW_AUTO_SWITCH_TO_MTLS_URL, False) self.assertEqual(client._connection.API_BASE_URL, "http://foo") + def test_ctor_w_emulator_wo_project(self): + from google.auth.credentials import AnonymousCredentials + from google.cloud.storage._helpers import STORAGE_EMULATOR_ENV_VAR + + # avoids authentication if STORAGE_EMULATOR_ENV_VAR is set + host = "http://localhost:8080" + environ = {STORAGE_EMULATOR_ENV_VAR: host} + with mock.patch("os.environ", environ): + client = self._make_one() + + self.assertIsNone(client.project) + self.assertEqual(client._connection.API_BASE_URL, host) + self.assertIsInstance(client._connection.credentials, AnonymousCredentials) + + # avoids authentication if storage emulator is set through api_endpoint + client = self._make_one( + client_options={"api_endpoint": "http://localhost:8080"} + ) + self.assertIsNone(client.project) + self.assertEqual(client._connection.API_BASE_URL, host) + self.assertIsInstance(client._connection.credentials, AnonymousCredentials) + + def test_ctor_w_emulator_w_environ_project(self): + from google.auth.credentials import AnonymousCredentials + from google.cloud.storage._helpers import STORAGE_EMULATOR_ENV_VAR + + # avoids authentication and infers the project from the environment + host = "http://localhost:8080" + environ_project = "environ-project" + environ = { + STORAGE_EMULATOR_ENV_VAR: host, + "GOOGLE_CLOUD_PROJECT": environ_project, + } + with mock.patch("os.environ", environ): + client = self._make_one() + + self.assertEqual(client.project, environ_project) + self.assertEqual(client._connection.API_BASE_URL, host) + self.assertIsInstance(client._connection.credentials, AnonymousCredentials) + + def test_ctor_w_emulator_w_project_arg(self): + from google.auth.credentials import AnonymousCredentials + from google.cloud.storage._helpers import STORAGE_EMULATOR_ENV_VAR + + # project argument overrides project set in the enviroment + host = "http://localhost:8080" + environ_project = "environ-project" + project = "my-test-project" + environ = { + STORAGE_EMULATOR_ENV_VAR: host, + "GOOGLE_CLOUD_PROJECT": environ_project, + } + with mock.patch("os.environ", environ): + client = self._make_one(project=project) + + self.assertEqual(client.project, project) + self.assertEqual(client._connection.API_BASE_URL, host) + self.assertIsInstance(client._connection.credentials, AnonymousCredentials) + def test_create_anonymous_client(self): from google.auth.credentials import AnonymousCredentials from google.cloud.storage._http import Connection From 46c3e66862b536393503ad213a30670f2f5e752e Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Tue, 18 Jan 2022 20:59:28 -0500 Subject: [PATCH 8/9] chore(python): Noxfile recognizes that tests can live in a folder (#696) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(python): Noxfile recognizes that tests can live in a folder Source-Link: https://github.com/googleapis/synthtool/commit/4760d8dce1351d93658cb11d02a1b7ceb23ae5d7 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:f0e4b51deef56bed74d3e2359c583fc104a8d6367da3984fc5c66938db738828 * update owlbot.py to remove python 3.6 samples * 🦉 Updates from OwlBot See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * 🦉 Updates from OwlBot See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md Co-authored-by: Owl Bot Co-authored-by: Anthonios Partheniou --- .github/.OwlBot.lock.yaml | 2 +- .github/CODEOWNERS | 12 +++++------- owlbot.py | 2 +- samples/snippets/noxfile.py | 1 + 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index eecb84c21..52d79c11f 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -1,3 +1,3 @@ docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:ae600f36b6bc972b368367b6f83a1d91ec2c82a4a116b383d67d547c56fe6de3 + digest: sha256:f0e4b51deef56bed74d3e2359c583fc104a8d6367da3984fc5c66938db738828 diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 4e9e8be82..b37686f76 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -3,12 +3,10 @@ # # For syntax help see: # https://help.github.com/en/github/creating-cloning-and-archiving-repositories/about-code-owners#codeowners-syntax +# Note: This file is autogenerated. To make changes to the codeowner team, please update .repo-metadata.json. +# @googleapis/yoshi-python @googleapis/cloud-storage-dpe are the default owners for changes in this repo +* @googleapis/yoshi-python @googleapis/cloud-storage-dpe -# The cloud-storage-dpe team is the default owner for anything not -# explicitly taken by someone else. -* @googleapis/cloud-storage-dpe @googleapis/yoshi-python - -# Additionally, the python-samples-owners team is also among -# the default owners for samples changes. -/samples/ @googleapis/cloud-storage-dpe @googleapis/yoshi-python @googleapis/python-samples-owners \ No newline at end of file +# @googleapis/python-samples-reviewers @googleapis/cloud-storage-dpe are the default owners for samples changes +/samples/ @googleapis/python-samples-reviewers @googleapis/cloud-storage-dpe diff --git a/owlbot.py b/owlbot.py index c4d3b0ccb..b6c08a888 100644 --- a/owlbot.py +++ b/owlbot.py @@ -45,7 +45,7 @@ "noxfile.py", "renovate.json", # do not bundle reports "CONTRIBUTING.rst", - ".github/CODEOWNERS", + ".kokoro/samples/python3.6", # remove python 3.6 support ], ) diff --git a/samples/snippets/noxfile.py b/samples/snippets/noxfile.py index 3bbef5d54..20cdfc620 100644 --- a/samples/snippets/noxfile.py +++ b/samples/snippets/noxfile.py @@ -187,6 +187,7 @@ def _session_tests( ) -> None: # check for presence of tests test_list = glob.glob("*_test.py") + glob.glob("test_*.py") + test_list.extend(glob.glob("tests")) if len(test_list) == 0: print("No tests found, skipping directory.") else: From 8622440f216a9de36698f6af67af249a52134bd7 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Wed, 19 Jan 2022 10:33:45 -0700 Subject: [PATCH 9/9] chore(main): release 2.1.0 (#695) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- CHANGELOG.md | 9 +++++++++ google/cloud/storage/version.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 19ff1db98..298620b4b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,15 @@ [1]: https://pypi.org/project/google-cloud-storage/#history +## [2.1.0](https://github.com/googleapis/python-storage/compare/v2.0.0...v2.1.0) (2022-01-19) + + +### Features + +* add turbo replication support and samples ([#622](https://github.com/googleapis/python-storage/issues/622)) ([4dafc81](https://github.com/googleapis/python-storage/commit/4dafc815470480ce9de7f0357e331d3fbd0ae9b7)) +* avoid authentication with storage emulator ([#679](https://github.com/googleapis/python-storage/issues/679)) ([8789afa](https://github.com/googleapis/python-storage/commit/8789afaaa1b2bd6f03fae72e3d87ce004ec10129)) +* remove python 3.6 support ([#689](https://github.com/googleapis/python-storage/issues/689)) ([8aa4130](https://github.com/googleapis/python-storage/commit/8aa4130ee068a1922161c8ca54a53a4a51d65ce0)) + ## [2.0.0](https://github.com/googleapis/python-storage/compare/v1.44.0...v2.0.0) (2022-01-12) diff --git a/google/cloud/storage/version.py b/google/cloud/storage/version.py index a12de3d25..8b5d3328c 100644 --- a/google/cloud/storage/version.py +++ b/google/cloud/storage/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.0.0" +__version__ = "2.1.0"