diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 757c9dca7..58fcbeeed 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:81ed5ecdfc7cac5b699ba4537376f3563f6f04122c4ec9e735d3b3dc1d43dd32 -# created: 2022-05-05T22:08:23.383410683Z + digest: sha256:c8878270182edaab99f2927969d4f700c3af265accd472c3425deedff2b7fd93 +# created: 2022-07-14T01:58:16.015625351Z diff --git a/.kokoro/continuous/prerelease-deps.cfg b/.kokoro/continuous/prerelease-deps.cfg new file mode 100644 index 000000000..3595fb43f --- /dev/null +++ b/.kokoro/continuous/prerelease-deps.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Only run this nox session. +env_vars: { + key: "NOX_SESSION" + value: "prerelease_deps" +} diff --git a/.kokoro/presubmit/prerelease-deps.cfg b/.kokoro/presubmit/prerelease-deps.cfg new file mode 100644 index 000000000..3595fb43f --- /dev/null +++ b/.kokoro/presubmit/prerelease-deps.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Only run this nox session. +env_vars: { + key: "NOX_SESSION" + value: "prerelease_deps" +} diff --git a/.kokoro/test-samples-impl.sh b/.kokoro/test-samples-impl.sh index 8a324c9c7..2c6500cae 100755 --- a/.kokoro/test-samples-impl.sh +++ b/.kokoro/test-samples-impl.sh @@ -33,7 +33,7 @@ export PYTHONUNBUFFERED=1 env | grep KOKORO # Install nox -python3.6 -m pip install --upgrade --quiet nox +python3.9 -m pip install --upgrade --quiet nox # Use secrets acessor service account to get secrets if [[ -f "${KOKORO_GFILE_DIR}/secrets_viewer_service_account.json" ]]; then @@ -76,7 +76,7 @@ for file in samples/**/requirements.txt; do echo "------------------------------------------------------------" # Use nox to execute the tests for the project. - python3.6 -m nox -s "$RUN_TESTS_SESSION" + python3.9 -m nox -s "$RUN_TESTS_SESSION" EXIT=$? # If this is a periodic build, send the test log to the FlakyBot. diff --git a/.repo-metadata.json b/.repo-metadata.json index 2cd2642fe..9e537d52f 100644 --- a/.repo-metadata.json +++ b/.repo-metadata.json @@ -13,5 +13,6 @@ "requires_billing": true, "default_version": "", "codeowner_team": "@googleapis/cloud-storage-dpe", - "api_shortname": "storage" + "api_shortname": "storage", + "api_description": "is a durable and highly available object storage service. Google Cloud Storage is almost infinitely scalable and guarantees consistency: when a write succeeds, the latest copy of the object will be returned to any GET, globally." } diff --git a/CHANGELOG.md b/CHANGELOG.md index 6bc2a1ea9..5c312a242 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,18 @@ [1]: https://pypi.org/project/google-cloud-storage/#history +## [2.5.0](https://github.com/googleapis/python-storage/compare/v2.4.0...v2.5.0) (2022-07-24) + + +### Features + +* Custom Placement Config Dual Region Support ([#819](https://github.com/googleapis/python-storage/issues/819)) ([febece7](https://github.com/googleapis/python-storage/commit/febece76802252278bb7626d931973a76561382a)) + + +### Documentation + +* open file-like objects in byte mode for uploads ([#824](https://github.com/googleapis/python-storage/issues/824)) ([4bd3d1d](https://github.com/googleapis/python-storage/commit/4bd3d1ddf21196b075bbd84cdcb553c5d7355b93)) + ## [2.4.0](https://github.com/googleapis/python-storage/compare/v2.3.0...v2.4.0) (2022-06-07) diff --git a/README.rst b/README.rst index 8a1304b73..3b2f84736 100644 --- a/README.rst +++ b/README.rst @@ -1,25 +1,22 @@ -Python Client for Google Cloud Storage -====================================== +Python Client for Google Cloud Storage API +========================================== -|GA| |pypi| |versions| +|stable| |pypi| |versions| -`Google Cloud Storage`_ allows you to store data on -Google infrastructure with very high reliability, performance and -availability, and can be used to distribute large data objects to users -via direct download. +`Google Cloud Storage API`_: is a durable and highly available object storage service. Google Cloud Storage is almost infinitely scalable and guarantees consistency: when a write succeeds, the latest copy of the object will be returned to any GET, globally. - `Client Library Documentation`_ -- `Storage API docs`_ +- `Product Documentation`_ -.. |GA| image:: https://img.shields.io/badge/support-GA-gold.svg - :target: https://github.com/googleapis/google-cloud-python/blob/main/README.rst#general-availability +.. |stable| image:: https://img.shields.io/badge/support-stable-gold.svg + :target: https://github.com/googleapis/google-cloud-python/blob/main/README.rst#stability-levels .. |pypi| image:: https://img.shields.io/pypi/v/google-cloud-storage.svg - :target: https://pypi.org/project/google-cloud-storage + :target: https://pypi.org/project/google-cloud-storage/ .. |versions| image:: https://img.shields.io/pypi/pyversions/google-cloud-storage.svg - :target: https://pypi.org/project/google-cloud-storage -.. _Google Cloud Storage: https://cloud.google.com/storage/docs -.. _Client Library Documentation: https://googleapis.dev/python/storage/latest -.. _Storage API docs: https://cloud.google.com/storage/docs/json_api/v1 + :target: https://pypi.org/project/google-cloud-storage/ +.. _Google Cloud Storage API: https://cloud.google.com/storage +.. _Client Library Documentation: https://cloud.google.com/python/docs/reference/storage/latest +.. _Product Documentation: https://cloud.google.com/storage Quick Start ----------- @@ -34,51 +31,56 @@ In order to use this library, you first need to go through the following steps: .. _Select or create a Cloud Platform project.: https://console.cloud.google.com/project .. _Enable billing for your project.: https://cloud.google.com/billing/docs/how-to/modify-project#enable_billing_for_a_project .. _Enable the Google Cloud Storage API.: https://cloud.google.com/storage -.. _Setup Authentication.: https://cloud.google.com/storage/docs/reference/libraries#setting_up_authentication +.. _Setup Authentication.: https://googleapis.dev/python/google-api-core/latest/auth.html Installation ~~~~~~~~~~~~ -`Set up a Python development environment`_ and install this library in a `venv`. -`venv`_ is a tool to create isolated Python environments. The basic problem it -addresses is one of dependencies and versions, and indirectly permissions. +Install this library in a `virtualenv`_ using pip. `virtualenv`_ is a tool to +create isolated Python environments. The basic problem it addresses is one of +dependencies and versions, and indirectly permissions. -Make sure you're using Python 3.7 or later, which includes `venv`_ by default. -With `venv`, it's possible to install this library without needing system +With `virtualenv`_, it's possible to install this library without needing system install permissions, and without clashing with the installed system dependencies. -.. _Set up a Python development environment: https://cloud.google.com/python/docs/setup -.. _`venv`: https://docs.python.org/3/library/venv.html +.. _`virtualenv`: https://virtualenv.pypa.io/en/latest/ + + +Code samples and snippets +~~~~~~~~~~~~~~~~~~~~~~~~~ + +Code samples and snippets live in the `samples/` folder. Supported Python Versions ^^^^^^^^^^^^^^^^^^^^^^^^^ +Our client libraries are compatible with all current `active`_ and `maintenance`_ versions of +Python. + Python >= 3.7 -Deprecated Python Versions -^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. _active: https://devguide.python.org/devcycle/#in-development-main-branch +.. _maintenance: https://devguide.python.org/devcycle/#maintenance-branches Unsupported Python Versions ^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Python <= 3.6 -Python == 3.6: the last released version which supported Python 3.6 was -``google-cloud-storage 2.0.0``, released 2022-01-12. +If you are using an `end-of-life`_ +version of Python, we recommend that you update as soon as possible to an actively supported version. -Python == 3.5: the last released version which supported Python 3.5 was -``google-cloud-storage 1.32.0``, released 2020-10-16. - -Python == 2.7: the last released version which supported Python 2.7 was -``google-cloud-storage 1.44.0``, released 2022-01-05. +.. _end-of-life: https://devguide.python.org/devcycle/#end-of-life-branches Mac/Linux ^^^^^^^^^ .. code-block:: console - python -m venv env - source env/bin/activate - pip install google-cloud-storage + pip install virtualenv + virtualenv + source /bin/activate + /bin/pip install google-cloud-storage Windows @@ -86,40 +88,20 @@ Windows .. code-block:: console - py -m venv env - .\env\Scripts\activate - pip install google-cloud-storage - - -Example Usage -~~~~~~~~~~~~~ - -.. code:: python - - # Imports the Google Cloud client library - from google.cloud import storage - - # Instantiates a client - client = storage.Client() - - # Creates a new bucket and uploads an object - new_bucket = client.create_bucket('new-bucket-id') - new_blob = new_bucket.blob('remote/path/storage.txt') - new_blob.upload_from_filename(filename='/local/path.txt') - - # Retrieve an existing bucket - # https://console.cloud.google.com/storage/browser/[bucket-id]/ - bucket = client.get_bucket('bucket-id') - # Then do other things... - blob = bucket.get_blob('remote/path/to/file.txt') - print(blob.download_as_bytes()) - blob.upload_from_string('New contents!') - + pip install virtualenv + virtualenv + \Scripts\activate + \Scripts\pip.exe install google-cloud-storage -What's Next -~~~~~~~~~~~ +Next Steps +~~~~~~~~~~ -Now that you've set up your Python client for Cloud Storage, -you can get started running `Storage samples.`_ +- Read the `Client Library Documentation`_ for Google Cloud Storage API + to see other available methods on the client. +- Read the `Google Cloud Storage API Product documentation`_ to learn + more about the product and see How-to Guides. +- View this `README`_ to see the full list of Cloud + APIs that we cover. -.. _Storage samples.: https://github.com/googleapis/python-storage/tree/main/samples +.. _Google Cloud Storage API Product documentation: https://cloud.google.com/storage +.. _README: https://github.com/googleapis/google-cloud-python/blob/main/README.rst diff --git a/google/cloud/storage/blob.py b/google/cloud/storage/blob.py index f47c09181..205d4aeb2 100644 --- a/google/cloud/storage/blob.py +++ b/google/cloud/storage/blob.py @@ -2456,7 +2456,7 @@ def upload_from_file( to that project. :type file_obj: file - :param file_obj: A file handle open for reading. + :param file_obj: A file handle opened in binary mode for reading. :type rewind: bool :param rewind: diff --git a/google/cloud/storage/bucket.py b/google/cloud/storage/bucket.py index 1b31baab7..5408b9373 100644 --- a/google/cloud/storage/bucket.py +++ b/google/cloud/storage/bucket.py @@ -2418,13 +2418,27 @@ def location(self, value): warnings.warn(_LOCATION_SETTER_MESSAGE, DeprecationWarning, stacklevel=2) self._location = value + @property + def data_locations(self): + """Retrieve the list of regional locations for custom dual-region buckets. + + See https://cloud.google.com/storage/docs/json_api/v1/buckets and + https://cloud.google.com/storage/docs/locations + + Returns ``None`` if the property has not been set before creation, + if the bucket's resource has not been loaded from the server, + or if the bucket is not a dual-regions bucket. + :rtype: list of str or ``NoneType`` + """ + custom_placement_config = self._properties.get("customPlacementConfig", {}) + return custom_placement_config.get("dataLocations") + @property def location_type(self): - """Retrieve or set the location type for the bucket. + """Retrieve the location type for the bucket. See https://cloud.google.com/storage/docs/storage-classes - :setter: Set the location type for this bucket. :getter: Gets the the location type for this bucket. :rtype: str or ``NoneType`` diff --git a/google/cloud/storage/client.py b/google/cloud/storage/client.py index a22b70f9a..acf675fbe 100644 --- a/google/cloud/storage/client.py +++ b/google/cloud/storage/client.py @@ -602,6 +602,7 @@ def _post_resource( google.cloud.exceptions.NotFound If the bucket is not found. """ + return self._connection.api_request( method="POST", path=path, @@ -847,6 +848,7 @@ def create_bucket( project=None, user_project=None, location=None, + data_locations=None, predefined_acl=None, predefined_default_object_acl=None, timeout=_DEFAULT_TIMEOUT, @@ -876,7 +878,11 @@ def create_bucket( location (str): (Optional) The location of the bucket. If not passed, the default location, US, will be used. If specifying a dual-region, - can be specified as a string, e.g., 'US-CENTRAL1+US-WEST1'. See: + `data_locations` should be set in conjunction.. See: + https://cloud.google.com/storage/docs/locations + data_locations (list of str): + (Optional) The list of regional locations of a custom dual-region bucket. + Dual-regions require exactly 2 regional locations. See: https://cloud.google.com/storage/docs/locations predefined_acl (str): (Optional) Name of predefined ACL to apply to bucket. See: @@ -979,6 +985,9 @@ def create_bucket( if location is not None: properties["location"] = location + if data_locations is not None: + properties["customPlacementConfig"] = {"dataLocations": data_locations} + api_response = self._post_resource( "/b", properties, diff --git a/google/cloud/storage/version.py b/google/cloud/storage/version.py index fe11624d9..5836d8051 100644 --- a/google/cloud/storage/version.py +++ b/google/cloud/storage/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.4.0" +__version__ = "2.5.0" diff --git a/samples/README.md b/samples/README.md index 2d9080067..173b60eae 100644 --- a/samples/README.md +++ b/samples/README.md @@ -324,7 +324,7 @@ View the [source code](https://github.com/googleapis/python-storage/blob/main/sa View the [source code](https://github.com/googleapis/python-storage/blob/main/samples/snippets/storage_create_bucket_dual_region.py). To run this sample: -`python storage_create_bucket_dual_region.py ` +`python storage_create_bucket_dual_region.py ` ----- ### Create Bucket Notifications diff --git a/samples/snippets/noxfile.py b/samples/snippets/noxfile.py index 38bb0a572..5fcb9d746 100644 --- a/samples/snippets/noxfile.py +++ b/samples/snippets/noxfile.py @@ -89,7 +89,7 @@ def get_pytest_env_vars() -> Dict[str, str]: # DO NOT EDIT - automatically generated. # All versions used to test samples. -ALL_VERSIONS = ["3.6", "3.7", "3.8", "3.9", "3.10"] +ALL_VERSIONS = ["3.7", "3.8", "3.9", "3.10"] # Any default versions that should be ignored. IGNORED_VERSIONS = TEST_CONFIG["ignored_versions"] diff --git a/samples/snippets/requirements-test.txt b/samples/snippets/requirements-test.txt index ed8c5dde8..077bdf929 100644 --- a/samples/snippets/requirements-test.txt +++ b/samples/snippets/requirements-test.txt @@ -1,3 +1,3 @@ pytest==7.1.2 mock==4.0.3 -backoff==2.1.0 \ No newline at end of file +backoff==2.1.2 \ No newline at end of file diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 44c1b701d..fe1ba5907 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,4 +1,4 @@ -google-cloud-pubsub==2.13.0 -google-cloud-storage==2.3.0 +google-cloud-pubsub==2.13.1 +google-cloud-storage==2.4.0 pandas===1.3.5; python_version == '3.7' -pandas==1.4.2; python_version >= '3.8' +pandas==1.4.3; python_version >= '3.8' diff --git a/samples/snippets/snippets_test.py b/samples/snippets/snippets_test.py index bdd8c528e..d0fefd488 100644 --- a/samples/snippets/snippets_test.py +++ b/samples/snippets/snippets_test.py @@ -435,10 +435,11 @@ def test_create_bucket_class_location(test_bucket_create): def test_create_bucket_dual_region(test_bucket_create, capsys): + location = "US" region_1 = "US-EAST1" region_2 = "US-WEST1" storage_create_bucket_dual_region.create_bucket_dual_region( - test_bucket_create.name, region_1, region_2 + test_bucket_create.name, location, region_1, region_2 ) out, _ = capsys.readouterr() assert f"Bucket {test_bucket_create.name} created in {region_1}+{region_2}" in out diff --git a/samples/snippets/storage_create_bucket_dual_region.py b/samples/snippets/storage_create_bucket_dual_region.py index e6f4ac01f..061f4c1db 100644 --- a/samples/snippets/storage_create_bucket_dual_region.py +++ b/samples/snippets/storage_create_bucket_dual_region.py @@ -24,8 +24,8 @@ from google.cloud import storage -def create_bucket_dual_region(bucket_name, region_1, region_2): - """Creates a Dual-Region Bucket with provided locations.""" +def create_bucket_dual_region(bucket_name, location, region_1, region_2): + """Creates a Dual-Region Bucket with provided location and regions..""" # The ID of your GCS bucket # bucket_name = "your-bucket-name" @@ -34,9 +34,10 @@ def create_bucket_dual_region(bucket_name, region_1, region_2): # https://cloud.google.com/storage/docs/locations # region_1 = "US-EAST1" # region_2 = "US-WEST1" + # location = "US" storage_client = storage.Client() - storage_client.create_bucket(bucket_name, location=f"{region_1}+{region_2}") + storage_client.create_bucket(bucket_name, location=location, data_locations=[region_1, region_2]) print(f"Bucket {bucket_name} created in {region_1}+{region_2}.") @@ -46,5 +47,5 @@ def create_bucket_dual_region(bucket_name, region_1, region_2): if __name__ == "__main__": create_bucket_dual_region( - bucket_name=sys.argv[1], region_1=sys.argv[2], region_2=sys.argv[3] + bucket_name=sys.argv[1], location=sys.argv[2], region_1=sys.argv[3], region_2=sys.argv[4] ) diff --git a/samples/snippets/storage_upload_from_stream.py b/samples/snippets/storage_upload_from_stream.py index e2d31a5e3..08eb25889 100644 --- a/samples/snippets/storage_upload_from_stream.py +++ b/samples/snippets/storage_upload_from_stream.py @@ -25,8 +25,8 @@ def upload_blob_from_stream(bucket_name, file_obj, destination_blob_name): # The stream or file (file-like object) from which to read # import io - # file_obj = io.StringIO() - # file_obj.write("This is test data.") + # file_obj = io.BytesIO() + # file_obj.write(b"This is test data.") # The desired name of the uploaded GCS object (blob) # destination_blob_name = "storage-object-name" diff --git a/scripts/readme-gen/templates/install_deps.tmpl.rst b/scripts/readme-gen/templates/install_deps.tmpl.rst index 275d64989..6f069c6c8 100644 --- a/scripts/readme-gen/templates/install_deps.tmpl.rst +++ b/scripts/readme-gen/templates/install_deps.tmpl.rst @@ -12,7 +12,7 @@ Install Dependencies .. _Python Development Environment Setup Guide: https://cloud.google.com/python/setup -#. Create a virtualenv. Samples are compatible with Python 3.6+. +#. Create a virtualenv. Samples are compatible with Python 3.7+. .. code-block:: bash diff --git a/tests/system/test_client.py b/tests/system/test_client.py index 9d9526a03..db912561d 100644 --- a/tests/system/test_client.py +++ b/tests/system/test_client.py @@ -68,21 +68,21 @@ def test_create_bucket_dual_region(storage_client, buckets_to_delete): from google.cloud.storage.constants import DUAL_REGION_LOCATION_TYPE new_bucket_name = _helpers.unique_name("dual-region-bucket") - region_1 = "US-EAST1" - region_2 = "US-WEST1" - dual_region = f"{region_1}+{region_2}" + location = "US" + data_locations = ["US-EAST1", "US-WEST1"] with pytest.raises(exceptions.NotFound): storage_client.get_bucket(new_bucket_name) created = _helpers.retry_429_503(storage_client.create_bucket)( - new_bucket_name, location=dual_region + new_bucket_name, location=location, data_locations=data_locations ) buckets_to_delete.append(created) assert created.name == new_bucket_name - assert created.location == dual_region + assert created.location == location assert created.location_type == DUAL_REGION_LOCATION_TYPE + assert created.data_locations == data_locations def test_list_buckets(storage_client, buckets_to_delete): diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index 07d1b0655..6769f3020 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -1451,6 +1451,44 @@ def test_create_bucket_w_explicit_location(self): _target_object=bucket, ) + def test_create_bucket_w_custom_dual_region(self): + project = "PROJECT" + bucket_name = "bucket-name" + location = "US" + data_locations = ["US-EAST1", "US-WEST1"] + api_response = { + "location": location, + "customPlacementConfig": {"dataLocations": data_locations}, + "name": bucket_name, + } + credentials = _make_credentials() + client = self._make_one(project=project, credentials=credentials) + client._post_resource = mock.Mock() + client._post_resource.return_value = api_response + + bucket = client.create_bucket( + bucket_name, location=location, data_locations=data_locations + ) + + self.assertEqual(bucket.location, location) + self.assertEqual(bucket.data_locations, data_locations) + + expected_path = "/b" + expected_data = { + "location": location, + "customPlacementConfig": {"dataLocations": data_locations}, + "name": bucket_name, + } + expected_query_params = {"project": project} + client._post_resource.assert_called_once_with( + expected_path, + expected_data, + query_params=expected_query_params, + timeout=self._get_default_timeout(), + retry=DEFAULT_RETRY, + _target_object=bucket, + ) + def test_create_bucket_w_explicit_project(self): project = "PROJECT" other_project = "other-project-123"