diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index dd98abbde..7f291dbd5 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:08e34975760f002746b1d8c86fdc90660be45945ee6d9db914d1508acdf9a547 -# created: 2023-10-09T14:06:13.397766266Z + digest: sha256:4f9b3b106ad0beafc2c8a415e3f62c1a0cc23cabea115dbe841b848f581cfe99 +# created: 2023-10-18T20:26:37.410353675Z diff --git a/.kokoro/requirements.txt b/.kokoro/requirements.txt index 0332d3267..16170d0ca 100644 --- a/.kokoro/requirements.txt +++ b/.kokoro/requirements.txt @@ -467,9 +467,9 @@ typing-extensions==4.4.0 \ --hash=sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa \ --hash=sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e # via -r requirements.in -urllib3==1.26.17 \ - --hash=sha256:24d6a242c28d29af46c3fae832c36db3bbebcc533dd1bb549172cd739c82df21 \ - --hash=sha256:94a757d178c9be92ef5539b8840d48dc9cf1b2709c9d6b588232a055c524458b +urllib3==1.26.18 \ + --hash=sha256:34b97092d7e0a3a8cf7cd10e386f401b3737364026c45e622aa02903dffe0f07 \ + --hash=sha256:f8ecc1bba5667413457c529ab955bf8c67b45db799d159066261719e328580a0 # via # requests # twine diff --git a/CHANGELOG.md b/CHANGELOG.md index 57d91347a..9a2f34ecb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,26 @@ [1]: https://pypi.org/project/google-cloud-storage/#history +## [2.13.0](https://github.com/googleapis/python-storage/compare/v2.12.0...v2.13.0) (2023-10-31) + + +### Features + +* Add Autoclass v2.1 support ([#1117](https://github.com/googleapis/python-storage/issues/1117)) ([d38adb6](https://github.com/googleapis/python-storage/commit/d38adb6a3136152ad68ad8a9c4583d06509307b2)) +* Add support for custom headers ([#1121](https://github.com/googleapis/python-storage/issues/1121)) ([2f92c3a](https://github.com/googleapis/python-storage/commit/2f92c3a2a3a1585d0f77be8fe3c2c5324140b71a)) + + +### Bug Fixes + +* Blob.from_string parse storage uri with regex ([#1170](https://github.com/googleapis/python-storage/issues/1170)) ([0a243fa](https://github.com/googleapis/python-storage/commit/0a243faf5d6ca89b977ea1cf543356e0dd04df95)) +* Bucket.delete(force=True) now works with version-enabled buckets ([#1172](https://github.com/googleapis/python-storage/issues/1172)) ([0de09d3](https://github.com/googleapis/python-storage/commit/0de09d30ea6083d962be1c1f5341ea14a2456dc7)) +* Fix typo in Bucket.clear_lifecycle_rules() ([#1169](https://github.com/googleapis/python-storage/issues/1169)) ([eae9ebe](https://github.com/googleapis/python-storage/commit/eae9ebed12d26832405c2f29fbdb14b4babf080d)) + + +### Documentation + +* Fix exception field in tm reference docs ([#1164](https://github.com/googleapis/python-storage/issues/1164)) ([eac91cb](https://github.com/googleapis/python-storage/commit/eac91cb6ffb0066248f824fc1f307140dd7c85da)) + ## [2.12.0](https://github.com/googleapis/python-storage/compare/v2.11.0...v2.12.0) (2023-10-12) diff --git a/google/cloud/storage/blob.py b/google/cloud/storage/blob.py index a95e08911..33998f81a 100644 --- a/google/cloud/storage/blob.py +++ b/google/cloud/storage/blob.py @@ -134,7 +134,9 @@ "Blob.download_as_string() is deprecated and will be removed in future. " "Use Blob.download_as_bytes() instead." ) - +_GS_URL_REGEX_PATTERN = re.compile( + r"(?Pgs)://(?P[a-z0-9_.-]+)/(?P.+)" +) _DEFAULT_CHUNKSIZE = 104857600 # 1024 * 1024 B * 100 = 100 MB _MAX_MULTIPART_SIZE = 8388608 # 8 MB @@ -403,12 +405,11 @@ def from_string(cls, uri, client=None): """ from google.cloud.storage.bucket import Bucket - scheme, netloc, path, query, frag = urlsplit(uri) - if scheme != "gs": + match = _GS_URL_REGEX_PATTERN.match(uri) + if not match: raise ValueError("URI scheme must be gs") - - bucket = Bucket(client, name=netloc) - return cls(path[1:], bucket) + bucket = Bucket(client, name=match.group("bucket_name")) + return cls(match.group("object_name"), bucket) def generate_signed_url( self, @@ -1738,11 +1739,13 @@ def _get_upload_arguments(self, client, content_type, filename=None, command=Non * The ``content_type`` as a string (according to precedence) """ content_type = self._get_content_type(content_type, filename=filename) + # Add any client attached custom headers to the upload headers. headers = { **_get_default_headers( client._connection.user_agent, content_type, command=command ), **_get_encryption_headers(self._encryption_key), + **client._extra_headers, } object_metadata = self._get_writable_metadata() return headers, object_metadata, content_type @@ -4313,9 +4316,11 @@ def _prep_and_do_download( if_etag_match=if_etag_match, if_etag_not_match=if_etag_not_match, ) + # Add any client attached custom headers to be sent with the request. headers = { **_get_default_headers(client._connection.user_agent, command=command), **headers, + **client._extra_headers, } transport = client._http diff --git a/google/cloud/storage/bucket.py b/google/cloud/storage/bucket.py index f6d5e5aa2..de3b2502e 100644 --- a/google/cloud/storage/bucket.py +++ b/google/cloud/storage/bucket.py @@ -1539,6 +1539,7 @@ def delete( client=client, timeout=timeout, retry=retry, + versions=True, ) ) if len(blobs) > self._MAX_OBJECTS_FOR_ITERATION: @@ -1557,6 +1558,7 @@ def delete( client=client, timeout=timeout, retry=retry, + preserve_generation=True, ) # We intentionally pass `_target_object=None` since a DELETE @@ -2275,7 +2277,7 @@ def lifecycle_rules(self, rules): rules = [dict(rule) for rule in rules] # Convert helpers if needed self._patch_property("lifecycle", {"rule": rules}) - def clear_lifecyle_rules(self): + def clear_lifecycle_rules(self): """Clear lifecycle rules configured for this bucket. See https://cloud.google.com/storage/docs/lifecycle and @@ -2283,6 +2285,10 @@ def clear_lifecyle_rules(self): """ self.lifecycle_rules = [] + def clear_lifecyle_rules(self): + """Deprecated alias for clear_lifecycle_rules.""" + return self.clear_lifecycle_rules() + def add_lifecycle_delete_rule(self, **kw): """Add a "delete" rule to lifecycle rules configured for this bucket. @@ -2685,13 +2691,10 @@ def autoclass_enabled(self, value): :type value: convertible to boolean :param value: If true, enable Autoclass for this bucket. If false, disable Autoclass for this bucket. - - .. note:: - To enable autoclass, you must set it at bucket creation time. - Currently, only patch requests that disable autoclass are supported. - """ - self._patch_property("autoclass", {"enabled": bool(value)}) + autoclass = self._properties.get("autoclass", {}) + autoclass["enabled"] = bool(value) + self._patch_property("autoclass", autoclass) @property def autoclass_toggle_time(self): @@ -2705,6 +2708,48 @@ def autoclass_toggle_time(self): if timestamp is not None: return _rfc3339_nanos_to_datetime(timestamp) + @property + def autoclass_terminal_storage_class(self): + """The storage class that objects in an Autoclass bucket eventually transition to if + they are not read for a certain length of time. Valid values are NEARLINE and ARCHIVE. + + See https://cloud.google.com/storage/docs/using-autoclass for details. + + :setter: Set the terminal storage class for Autoclass configuration. + :getter: Get the terminal storage class for Autoclass configuration. + + :rtype: str + :returns: The terminal storage class if Autoclass is enabled, else ``None``. + """ + autoclass = self._properties.get("autoclass", {}) + return autoclass.get("terminalStorageClass", None) + + @autoclass_terminal_storage_class.setter + def autoclass_terminal_storage_class(self, value): + """The storage class that objects in an Autoclass bucket eventually transition to if + they are not read for a certain length of time. Valid values are NEARLINE and ARCHIVE. + + See https://cloud.google.com/storage/docs/using-autoclass for details. + + :type value: str + :param value: The only valid values are `"NEARLINE"` and `"ARCHIVE"`. + """ + autoclass = self._properties.get("autoclass", {}) + autoclass["terminalStorageClass"] = value + self._patch_property("autoclass", autoclass) + + @property + def autoclass_terminal_storage_class_update_time(self): + """The time at which the Autoclass terminal_storage_class field was last updated for this bucket + :rtype: datetime.datetime or ``NoneType`` + :returns: point-in time at which the bucket's terminal_storage_class is last updated, or ``None`` if the property is not set locally. + """ + autoclass = self._properties.get("autoclass") + if autoclass is not None: + timestamp = autoclass.get("terminalStorageClassUpdateTime") + if timestamp is not None: + return _rfc3339_nanos_to_datetime(timestamp) + def configure_website(self, main_page_suffix=None, not_found_page=None): """Configure website-related properties. diff --git a/google/cloud/storage/client.py b/google/cloud/storage/client.py index 10f2e5904..eea889f67 100644 --- a/google/cloud/storage/client.py +++ b/google/cloud/storage/client.py @@ -94,6 +94,11 @@ class Client(ClientWithProject): (Optional) Whether authentication is required under custom endpoints. If false, uses AnonymousCredentials and bypasses authentication. Defaults to True. Note this is only used when a custom endpoint is set in conjunction. + + :type extra_headers: dict + :param extra_headers: + (Optional) Custom headers to be sent with the requests attached to the client. + For example, you can add custom audit logging headers. """ SCOPE = ( @@ -111,6 +116,7 @@ def __init__( client_info=None, client_options=None, use_auth_w_custom_endpoint=True, + extra_headers={}, ): self._base_connection = None @@ -127,6 +133,7 @@ def __init__( # are passed along, for use in __reduce__ defined elsewhere. self._initial_client_info = client_info self._initial_client_options = client_options + self._extra_headers = extra_headers kw_args = {"client_info": client_info} @@ -172,7 +179,10 @@ def __init__( if no_project: self.project = None - self._connection = Connection(self, **kw_args) + # Pass extra_headers to Connection + connection = Connection(self, **kw_args) + connection.extra_headers = extra_headers + self._connection = connection self._batch_stack = _LocalStack() @classmethod diff --git a/google/cloud/storage/transfer_manager.py b/google/cloud/storage/transfer_manager.py index 41a67b5a4..8190f844d 100644 --- a/google/cloud/storage/transfer_manager.py +++ b/google/cloud/storage/transfer_manager.py @@ -865,11 +865,11 @@ def download_chunks_concurrently( :raises: :exc:`concurrent.futures.TimeoutError` if deadline is exceeded. - :exc:`google.resumable_media.common.DataCorruption` if the download's - checksum doesn't agree with server-computed checksum. The - `google.resumable_media` exception is used here for consistency - with other download methods despite the exception originating - elsewhere. + :exc:`google.resumable_media.common.DataCorruption` + if the download's checksum doesn't agree with server-computed + checksum. The `google.resumable_media` exception is used here for + consistency with other download methods despite the exception + originating elsewhere. """ client = blob.client @@ -1289,6 +1289,7 @@ def _reduce_client(cl): _http = None # Can't carry this over client_info = cl._initial_client_info client_options = cl._initial_client_options + extra_headers = cl._extra_headers return _LazyClient, ( client_object_id, @@ -1297,6 +1298,7 @@ def _reduce_client(cl): _http, client_info, client_options, + extra_headers, ) diff --git a/google/cloud/storage/version.py b/google/cloud/storage/version.py index 67e043bde..b6000e20f 100644 --- a/google/cloud/storage/version.py +++ b/google/cloud/storage/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.12.0" +__version__ = "2.13.0" diff --git a/samples/snippets/snippets_test.py b/samples/snippets/snippets_test.py index 7a5f8c960..7add15184 100644 --- a/samples/snippets/snippets_test.py +++ b/samples/snippets/snippets_test.py @@ -449,23 +449,27 @@ def test_get_set_autoclass(new_bucket_obj, test_bucket, capsys): out, _ = capsys.readouterr() assert "Autoclass enabled is set to False" in out assert bucket.autoclass_toggle_time is None + assert bucket.autoclass_terminal_storage_class_update_time is None # Test enabling Autoclass at bucket creation new_bucket_obj.autoclass_enabled = True bucket = storage.Client().create_bucket(new_bucket_obj) assert bucket.autoclass_enabled is True + assert bucket.autoclass_terminal_storage_class == "NEARLINE" - # Test disabling Autoclass - bucket = storage_set_autoclass.set_autoclass(bucket.name, False) + # Test set terminal_storage_class to ARCHIVE + bucket = storage_set_autoclass.set_autoclass(bucket.name) out, _ = capsys.readouterr() - assert "Autoclass enabled is set to False" in out - assert bucket.autoclass_enabled is False + assert "Autoclass enabled is set to True" in out + assert bucket.autoclass_enabled is True + assert bucket.autoclass_terminal_storage_class == "ARCHIVE" # Test get Autoclass bucket = storage_get_autoclass.get_autoclass(bucket.name) out, _ = capsys.readouterr() - assert "Autoclass enabled is set to False" in out + assert "Autoclass enabled is set to True" in out assert bucket.autoclass_toggle_time is not None + assert bucket.autoclass_terminal_storage_class_update_time is not None def test_bucket_lifecycle_management(test_bucket, capsys): diff --git a/samples/snippets/storage_get_autoclass.py b/samples/snippets/storage_get_autoclass.py index d4bcbf3f4..30fa0c4f6 100644 --- a/samples/snippets/storage_get_autoclass.py +++ b/samples/snippets/storage_get_autoclass.py @@ -29,8 +29,11 @@ def get_autoclass(bucket_name): bucket = storage_client.get_bucket(bucket_name) autoclass_enabled = bucket.autoclass_enabled autoclass_toggle_time = bucket.autoclass_toggle_time + terminal_storage_class = bucket.autoclass_terminal_storage_class + tsc_update_time = bucket.autoclass_terminal_storage_class_update_time print(f"Autoclass enabled is set to {autoclass_enabled} for {bucket.name} at {autoclass_toggle_time}.") + print(f"Autoclass terminal storage class is set to {terminal_storage_class} for {bucket.name} at {tsc_update_time}.") return bucket diff --git a/samples/snippets/storage_set_autoclass.py b/samples/snippets/storage_set_autoclass.py index a25151f3b..eec5a550f 100644 --- a/samples/snippets/storage_set_autoclass.py +++ b/samples/snippets/storage_set_autoclass.py @@ -20,23 +20,27 @@ from google.cloud import storage -def set_autoclass(bucket_name, toggle): - """Disable Autoclass for a bucket. +def set_autoclass(bucket_name): + """Configure the Autoclass setting for a bucket. - Note: Only patch requests that disable autoclass are currently supported. - To enable autoclass, you must set it at bucket creation time. + terminal_storage_class field is optional and defaults to NEARLINE if not otherwise specified. + Valid terminal_storage_class values are NEARLINE and ARCHIVE. """ # The ID of your GCS bucket # bucket_name = "my-bucket" - # Boolean toggle - if true, enables Autoclass; if false, disables Autoclass - # toggle = False + # Enable Autoclass for a bucket. Set enabled to false to disable Autoclass. + # Set Autoclass.TerminalStorageClass, valid values are NEARLINE and ARCHIVE. + enabled = True + terminal_storage_class = "ARCHIVE" storage_client = storage.Client() bucket = storage_client.bucket(bucket_name) - bucket.autoclass_enabled = toggle + bucket.autoclass_enabled = enabled + bucket.autoclass_terminal_storage_class = terminal_storage_class bucket.patch() print(f"Autoclass enabled is set to {bucket.autoclass_enabled} for {bucket.name} at {bucket.autoclass_toggle_time}.") + print(f"Autoclass terminal storage class is {bucket.autoclass_terminal_storage_class}.") return bucket @@ -44,4 +48,4 @@ def set_autoclass(bucket_name, toggle): # [END storage_set_autoclass] if __name__ == "__main__": - set_autoclass(bucket_name=sys.argv[1], toggle=sys.argv[2]) + set_autoclass(bucket_name=sys.argv[1]) diff --git a/tests/system/test_bucket.py b/tests/system/test_bucket.py index ac949cf96..19b21bac2 100644 --- a/tests/system/test_bucket.py +++ b/tests/system/test_bucket.py @@ -1047,7 +1047,9 @@ def test_new_bucket_with_autoclass( storage_client, buckets_to_delete, ): - # Autoclass can be enabled/disabled via bucket create + from google.cloud.storage import constants + + # Autoclass can be enabled via bucket create bucket_name = _helpers.unique_name("new-w-autoclass") bucket_obj = storage_client.bucket(bucket_name) bucket_obj.autoclass_enabled = True @@ -1055,7 +1057,9 @@ def test_new_bucket_with_autoclass( previous_toggle_time = bucket.autoclass_toggle_time buckets_to_delete.append(bucket) + # Autoclass terminal_storage_class is defaulted to NEARLINE if not specified assert bucket.autoclass_enabled is True + assert bucket.autoclass_terminal_storage_class == constants.NEARLINE_STORAGE_CLASS # Autoclass can be enabled/disabled via bucket patch bucket.autoclass_enabled = False @@ -1063,3 +1067,77 @@ def test_new_bucket_with_autoclass( assert bucket.autoclass_enabled is False assert bucket.autoclass_toggle_time != previous_toggle_time + + +def test_bucket_delete_force(storage_client): + bucket_name = _helpers.unique_name("version-disabled") + bucket_obj = storage_client.bucket(bucket_name) + bucket = storage_client.create_bucket(bucket_obj) + + BLOB_NAME = "my_object" + blob = bucket.blob(BLOB_NAME) + blob.upload_from_string("abcd") + blob.upload_from_string("efgh") + + blobs = bucket.list_blobs(versions=True) + counter = 0 + for blob in blobs: + counter += 1 + assert blob.name == BLOB_NAME + assert counter == 1 + + bucket.delete(force=True) # Will fail with 409 if blobs aren't deleted + + +def test_bucket_delete_force_works_with_versions(storage_client): + bucket_name = _helpers.unique_name("version-enabled") + bucket_obj = storage_client.bucket(bucket_name) + bucket_obj.versioning_enabled = True + bucket = storage_client.create_bucket(bucket_obj) + assert bucket.versioning_enabled + + BLOB_NAME = "my_versioned_object" + blob = bucket.blob(BLOB_NAME) + blob.upload_from_string("abcd") + blob.upload_from_string("efgh") + + blobs = bucket.list_blobs(versions=True) + counter = 0 + for blob in blobs: + counter += 1 + assert blob.name == BLOB_NAME + assert counter == 2 + + bucket.delete(force=True) # Will fail with 409 if versions aren't deleted + + +def test_config_autoclass_w_existing_bucket( + storage_client, + buckets_to_delete, +): + from google.cloud.storage import constants + + bucket_name = _helpers.unique_name("for-autoclass") + bucket = storage_client.create_bucket(bucket_name) + buckets_to_delete.append(bucket) + assert bucket.autoclass_enabled is False + assert bucket.autoclass_toggle_time is None + assert bucket.autoclass_terminal_storage_class is None + assert bucket.autoclass_terminal_storage_class_update_time is None + + # Enable Autoclass on existing buckets with terminal_storage_class set to ARCHIVE + bucket.autoclass_enabled = True + bucket.autoclass_terminal_storage_class = constants.ARCHIVE_STORAGE_CLASS + bucket.patch(if_metageneration_match=bucket.metageneration) + previous_tsc_update_time = bucket.autoclass_terminal_storage_class_update_time + assert bucket.autoclass_enabled is True + assert bucket.autoclass_terminal_storage_class == constants.ARCHIVE_STORAGE_CLASS + + # Configure Autoclass terminal_storage_class to NEARLINE + bucket.autoclass_terminal_storage_class = constants.NEARLINE_STORAGE_CLASS + bucket.patch(if_metageneration_match=bucket.metageneration) + assert bucket.autoclass_enabled is True + assert bucket.autoclass_terminal_storage_class == constants.NEARLINE_STORAGE_CLASS + assert ( + bucket.autoclass_terminal_storage_class_update_time != previous_tsc_update_time + ) diff --git a/tests/unit/test__http.py b/tests/unit/test__http.py index e64ae0bab..3ea3ed1a4 100644 --- a/tests/unit/test__http.py +++ b/tests/unit/test__http.py @@ -71,6 +71,55 @@ def test_extra_headers(self): timeout=_DEFAULT_TIMEOUT, ) + def test_metadata_op_has_client_custom_headers(self): + import requests + import google.auth.credentials + from google.cloud import _http as base_http + from google.cloud.storage import Client + from google.cloud.storage.constants import _DEFAULT_TIMEOUT + + custom_headers = { + "x-goog-custom-audit-foo": "bar", + "x-goog-custom-audit-user": "baz", + } + http = mock.create_autospec(requests.Session, instance=True) + response = requests.Response() + response.status_code = 200 + data = b"brent-spiner" + response._content = data + http.is_mtls = False + http.request.return_value = response + credentials = mock.Mock(spec=google.auth.credentials.Credentials) + client = Client( + project="project", + credentials=credentials, + _http=http, + extra_headers=custom_headers, + ) + req_data = "hey-yoooouuuuu-guuuuuyyssss" + with patch.object( + _helpers, "_get_invocation_id", return_value=GCCL_INVOCATION_TEST_CONST + ): + result = client._connection.api_request( + "GET", "/rainbow", data=req_data, expect_json=False + ) + self.assertEqual(result, data) + + expected_headers = { + **custom_headers, + "Accept-Encoding": "gzip", + base_http.CLIENT_INFO_HEADER: f"{client._connection.user_agent} {GCCL_INVOCATION_TEST_CONST}", + "User-Agent": client._connection.user_agent, + } + expected_uri = client._connection.build_api_url("/rainbow") + http.request.assert_called_once_with( + data=req_data, + headers=expected_headers, + method="GET", + url=expected_uri, + timeout=_DEFAULT_TIMEOUT, + ) + def test_build_api_url_no_extra_query_params(self): from urllib.parse import parse_qsl from urllib.parse import urlsplit diff --git a/tests/unit/test_blob.py b/tests/unit/test_blob.py index 1e84704b1..d5058e23c 100644 --- a/tests/unit/test_blob.py +++ b/tests/unit/test_blob.py @@ -2246,8 +2246,13 @@ def test__set_metadata_to_none(self): def test__get_upload_arguments(self): name = "blob-name" key = b"[pXw@,p@@AfBfrR3x-2b2SCHR,.?YwRO" + custom_headers = { + "x-goog-custom-audit-foo": "bar", + "x-goog-custom-audit-user": "baz", + } client = mock.Mock(_connection=_Connection) client._connection.user_agent = "testing 1.2.3" + client._extra_headers = custom_headers blob = self._make_one(name, bucket=None, encryption_key=key) blob.content_disposition = "inline" @@ -2271,6 +2276,7 @@ def test__get_upload_arguments(self): "X-Goog-Encryption-Algorithm": "AES256", "X-Goog-Encryption-Key": header_key_value, "X-Goog-Encryption-Key-Sha256": header_key_hash_value, + **custom_headers, } self.assertEqual( headers["X-Goog-API-Client"], @@ -2325,6 +2331,7 @@ def _do_multipart_success( client = mock.Mock(_http=transport, _connection=_Connection, spec=["_http"]) client._connection.API_BASE_URL = "https://storage.googleapis.com" + client._extra_headers = {} # Mock get_api_base_url_for_mtls function. mtls_url = "https://foo.mtls" @@ -2424,11 +2431,14 @@ def _do_multipart_success( with patch.object( _helpers, "_get_invocation_id", return_value=GCCL_INVOCATION_TEST_CONST ): - headers = _get_default_headers( - client._connection.user_agent, - b'multipart/related; boundary="==0=="', - "application/xml", - ) + headers = { + **_get_default_headers( + client._connection.user_agent, + b'multipart/related; boundary="==0=="', + "application/xml", + ), + **client._extra_headers, + } client._http.request.assert_called_once_with( "POST", upload_url, data=payload, headers=headers, timeout=expected_timeout ) @@ -2520,6 +2530,19 @@ def test__do_multipart_upload_with_client(self, mock_get_boundary): transport = self._mock_transport(http.client.OK, {}) client = mock.Mock(_http=transport, _connection=_Connection, spec=["_http"]) client._connection.API_BASE_URL = "https://storage.googleapis.com" + client._extra_headers = {} + self._do_multipart_success(mock_get_boundary, client=client) + + @mock.patch("google.resumable_media._upload.get_boundary", return_value=b"==0==") + def test__do_multipart_upload_with_client_custom_headers(self, mock_get_boundary): + custom_headers = { + "x-goog-custom-audit-foo": "bar", + "x-goog-custom-audit-user": "baz", + } + transport = self._mock_transport(http.client.OK, {}) + client = mock.Mock(_http=transport, _connection=_Connection, spec=["_http"]) + client._connection.API_BASE_URL = "https://storage.googleapis.com" + client._extra_headers = custom_headers self._do_multipart_success(mock_get_boundary, client=client) @mock.patch("google.resumable_media._upload.get_boundary", return_value=b"==0==") @@ -2597,6 +2620,7 @@ def _initiate_resumable_helper( # Create some mock arguments and call the method under test. client = mock.Mock(_http=transport, _connection=_Connection, spec=["_http"]) client._connection.API_BASE_URL = "https://storage.googleapis.com" + client._extra_headers = {} # Mock get_api_base_url_for_mtls function. mtls_url = "https://foo.mtls" @@ -2677,13 +2701,15 @@ def _initiate_resumable_helper( _helpers, "_get_invocation_id", return_value=GCCL_INVOCATION_TEST_CONST ): if extra_headers is None: - self.assertEqual( - upload._headers, - _get_default_headers(client._connection.user_agent, content_type), - ) + expected_headers = { + **_get_default_headers(client._connection.user_agent, content_type), + **client._extra_headers, + } + self.assertEqual(upload._headers, expected_headers) else: expected_headers = { **_get_default_headers(client._connection.user_agent, content_type), + **client._extra_headers, **extra_headers, } self.assertEqual(upload._headers, expected_headers) @@ -2730,9 +2756,12 @@ def _initiate_resumable_helper( with patch.object( _helpers, "_get_invocation_id", return_value=GCCL_INVOCATION_TEST_CONST ): - expected_headers = _get_default_headers( - client._connection.user_agent, x_upload_content_type=content_type - ) + expected_headers = { + **_get_default_headers( + client._connection.user_agent, x_upload_content_type=content_type + ), + **client._extra_headers, + } if size is not None: expected_headers["x-upload-content-length"] = str(size) if extra_headers is not None: @@ -2824,6 +2853,21 @@ def test__initiate_resumable_upload_with_client(self): client = mock.Mock(_http=transport, _connection=_Connection, spec=["_http"]) client._connection.API_BASE_URL = "https://storage.googleapis.com" + client._extra_headers = {} + self._initiate_resumable_helper(client=client) + + def test__initiate_resumable_upload_with_client_custom_headers(self): + custom_headers = { + "x-goog-custom-audit-foo": "bar", + "x-goog-custom-audit-user": "baz", + } + resumable_url = "http://test.invalid?upload_id=hey-you" + response_headers = {"location": resumable_url} + transport = self._mock_transport(http.client.OK, response_headers) + + client = mock.Mock(_http=transport, _connection=_Connection, spec=["_http"]) + client._connection.API_BASE_URL = "https://storage.googleapis.com" + client._extra_headers = custom_headers self._initiate_resumable_helper(client=client) def _make_resumable_transport( @@ -3000,6 +3044,7 @@ def _do_resumable_helper( client = mock.Mock(_http=transport, _connection=_Connection, spec=["_http"]) client._connection.API_BASE_URL = "https://storage.googleapis.com" client._connection.user_agent = USER_AGENT + client._extra_headers = {} stream = io.BytesIO(data) bucket = _Bucket(name="yesterday") @@ -3612,26 +3657,32 @@ def _create_resumable_upload_session_helper( if_metageneration_match=None, if_metageneration_not_match=None, retry=None, + client=None, ): bucket = _Bucket(name="alex-trebek") blob = self._make_one("blob-name", bucket=bucket) chunk_size = 99 * blob._CHUNK_SIZE_MULTIPLE blob.chunk_size = chunk_size - - # Create mocks to be checked for doing transport. resumable_url = "http://test.invalid?upload_id=clean-up-everybody" - response_headers = {"location": resumable_url} - transport = self._mock_transport(http.client.OK, response_headers) - if side_effect is not None: - transport.request.side_effect = side_effect - - # Create some mock arguments and call the method under test. content_type = "text/plain" size = 10000 - client = mock.Mock(_http=transport, _connection=_Connection, spec=["_http"]) - client._connection.API_BASE_URL = "https://storage.googleapis.com" - client._connection.user_agent = "testing 1.2.3" + transport = None + if not client: + # Create mocks to be checked for doing transport. + response_headers = {"location": resumable_url} + transport = self._mock_transport(http.client.OK, response_headers) + + # Create some mock arguments and call the method under test. + client = mock.Mock(_http=transport, _connection=_Connection, spec=["_http"]) + client._connection.API_BASE_URL = "https://storage.googleapis.com" + client._connection.user_agent = "testing 1.2.3" + client._extra_headers = {} + + if transport is None: + transport = client._http + if side_effect is not None: + transport.request.side_effect = side_effect if timeout is None: expected_timeout = self._get_default_timeout() timeout_kwarg = {} @@ -3689,6 +3740,7 @@ def _create_resumable_upload_session_helper( **_get_default_headers( client._connection.user_agent, x_upload_content_type=content_type ), + **client._extra_headers, "x-upload-content-length": str(size), "x-upload-content-type": content_type, } @@ -3750,6 +3802,28 @@ def test_create_resumable_upload_session_with_failure(self): self.assertIn(message, exc_info.exception.message) self.assertEqual(exc_info.exception.errors, []) + def test_create_resumable_upload_session_with_client(self): + resumable_url = "http://test.invalid?upload_id=clean-up-everybody" + response_headers = {"location": resumable_url} + transport = self._mock_transport(http.client.OK, response_headers) + client = mock.Mock(_http=transport, _connection=_Connection, spec=["_http"]) + client._connection.API_BASE_URL = "https://storage.googleapis.com" + client._extra_headers = {} + self._create_resumable_upload_session_helper(client=client) + + def test_create_resumable_upload_session_with_client_custom_headers(self): + custom_headers = { + "x-goog-custom-audit-foo": "bar", + "x-goog-custom-audit-user": "baz", + } + resumable_url = "http://test.invalid?upload_id=clean-up-everybody" + response_headers = {"location": resumable_url} + transport = self._mock_transport(http.client.OK, response_headers) + client = mock.Mock(_http=transport, _connection=_Connection, spec=["_http"]) + client._connection.API_BASE_URL = "https://storage.googleapis.com" + client._extra_headers = custom_headers + self._create_resumable_upload_session_helper(client=client) + def test_get_iam_policy_defaults(self): from google.cloud.storage.iam import STORAGE_OWNER_ROLE from google.cloud.storage.iam import STORAGE_EDITOR_ROLE @@ -5745,13 +5819,21 @@ def test_from_string_w_valid_uri(self): from google.cloud.storage.blob import Blob client = self._make_client() - uri = "gs://BUCKET_NAME/b" - blob = Blob.from_string(uri, client) + basic_uri = "gs://bucket_name/b" + blob = Blob.from_string(basic_uri, client) self.assertIsInstance(blob, Blob) self.assertIs(blob.client, client) self.assertEqual(blob.name, "b") - self.assertEqual(blob.bucket.name, "BUCKET_NAME") + self.assertEqual(blob.bucket.name, "bucket_name") + + nested_uri = "gs://bucket_name/path1/path2/b#name" + blob = Blob.from_string(nested_uri, client) + + self.assertIsInstance(blob, Blob) + self.assertIs(blob.client, client) + self.assertEqual(blob.name, "path1/path2/b#name") + self.assertEqual(blob.bucket.name, "bucket_name") def test_from_string_w_invalid_uri(self): from google.cloud.storage.blob import Blob @@ -5815,6 +5897,34 @@ def test_open(self): with self.assertRaises(ValueError): blob.open("w", ignore_flush=False) + def test_downloads_w_client_custom_headers(self): + import google.auth.credentials + from google.cloud.storage import Client + + custom_headers = { + "x-goog-custom-audit-foo": "bar", + "x-goog-custom-audit-user": "baz", + } + credentials = mock.Mock(spec=google.auth.credentials.Credentials) + client = Client( + project="project", credentials=credentials, extra_headers=custom_headers + ) + blob = self._make_one("blob-name", bucket=_Bucket(client)) + file_obj = io.BytesIO() + + downloads = { + client.download_blob_to_file: (blob, file_obj), + blob.download_to_file: (file_obj,), + blob.download_as_bytes: (), + } + for method, args in downloads.items(): + with mock.patch.object(blob, "_do_download"): + method(*args) + blob._do_download.assert_called() + called_headers = blob._do_download.call_args.args[-4] + self.assertIsInstance(called_headers, dict) + self.assertDictContainsSubset(custom_headers, called_headers) + class Test__quote(unittest.TestCase): @staticmethod diff --git a/tests/unit/test_bucket.py b/tests/unit/test_bucket.py index 0c0873ee4..8db6a2e62 100644 --- a/tests/unit/test_bucket.py +++ b/tests/unit/test_bucket.py @@ -1419,6 +1419,7 @@ def test_delete_hit_w_force_w_user_project_w_explicit_timeout_retry(self): client=client, timeout=timeout, retry=retry, + versions=True, ) bucket.delete_blobs.assert_called_once_with( @@ -1427,6 +1428,7 @@ def test_delete_hit_w_force_w_user_project_w_explicit_timeout_retry(self): client=client, timeout=timeout, retry=retry, + preserve_generation=True, ) expected_query_params = {"userProject": user_project} @@ -1456,6 +1458,7 @@ def test_delete_hit_w_force_delete_blobs(self): client=client, timeout=self._get_default_timeout(), retry=DEFAULT_RETRY, + versions=True, ) bucket.delete_blobs.assert_called_once_with( @@ -1464,6 +1467,7 @@ def test_delete_hit_w_force_delete_blobs(self): client=client, timeout=self._get_default_timeout(), retry=DEFAULT_RETRY, + preserve_generation=True, ) expected_query_params = {} @@ -1483,8 +1487,10 @@ def test_delete_w_force_w_user_project_w_miss_on_blob(self): client = mock.Mock(spec=["_delete_resource"]) client._delete_resource.return_value = None bucket = self._make_one(client=client, name=name) - blob = mock.Mock(spec=["name"]) + blob = mock.Mock(spec=["name", "generation"]) blob.name = blob_name + GEN = 1234 + blob.generation = GEN blobs = [blob] bucket.list_blobs = mock.Mock(return_value=iter(blobs)) bucket.delete_blob = mock.Mock(side_effect=NotFound("testing")) @@ -1496,7 +1502,7 @@ def test_delete_w_force_w_user_project_w_miss_on_blob(self): bucket.delete_blob.assert_called_once_with( blob_name, client=client, - generation=None, + generation=GEN, if_generation_match=None, if_generation_not_match=None, if_metageneration_match=None, @@ -2448,6 +2454,7 @@ def test_clear_lifecycle_rules(self): bucket._properties["lifecycle"] = {"rule": rules} self.assertEqual(list(bucket.lifecycle_rules), rules) + # This is a deprecated alias and will test both methods bucket.clear_lifecyle_rules() self.assertEqual(list(bucket.lifecycle_rules), []) @@ -2658,15 +2665,19 @@ def test_autoclass_enabled_getter_and_setter(self): self.assertIn("autoclass", bucket._changes) self.assertFalse(bucket.autoclass_enabled) - def test_autoclass_toggle_time_missing(self): + def test_autoclass_config_unset(self): bucket = self._make_one() self.assertIsNone(bucket.autoclass_toggle_time) + self.assertIsNone(bucket.autoclass_terminal_storage_class) + self.assertIsNone(bucket.autoclass_terminal_storage_class_update_time) properties = {"autoclass": {}} bucket = self._make_one(properties=properties) self.assertIsNone(bucket.autoclass_toggle_time) + self.assertIsNone(bucket.autoclass_terminal_storage_class) + self.assertIsNone(bucket.autoclass_terminal_storage_class_update_time) - def test_autoclass_toggle_time(self): + def test_autoclass_toggle_and_tsc_update_time(self): import datetime from google.cloud._helpers import _datetime_to_rfc3339 from google.cloud._helpers import UTC @@ -2676,10 +2687,31 @@ def test_autoclass_toggle_time(self): "autoclass": { "enabled": True, "toggleTime": _datetime_to_rfc3339(effective_time), + "terminalStorageClass": "NEARLINE", + "terminalStorageClassUpdateTime": _datetime_to_rfc3339(effective_time), } } bucket = self._make_one(properties=properties) self.assertEqual(bucket.autoclass_toggle_time, effective_time) + self.assertEqual( + bucket.autoclass_terminal_storage_class_update_time, effective_time + ) + + def test_autoclass_tsc_getter_and_setter(self): + from google.cloud.storage import constants + + properties = { + "autoclass": {"terminalStorageClass": constants.ARCHIVE_STORAGE_CLASS} + } + bucket = self._make_one(properties=properties) + self.assertEqual( + bucket.autoclass_terminal_storage_class, constants.ARCHIVE_STORAGE_CLASS + ) + bucket.autoclass_terminal_storage_class = constants.NEARLINE_STORAGE_CLASS + self.assertIn("autoclass", bucket._changes) + self.assertEqual( + bucket.autoclass_terminal_storage_class, constants.NEARLINE_STORAGE_CLASS + ) def test_get_logging_w_prefix(self): NAME = "name" diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index 0c1c5efee..4629ecf28 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -178,6 +178,18 @@ def test_ctor_w_client_options_object(self): client._connection.API_BASE_URL, "https://www.foo-googleapis.com" ) + def test_ctor_w_custom_headers(self): + PROJECT = "PROJECT" + credentials = _make_credentials() + custom_headers = {"x-goog-custom-audit-foo": "bar"} + client = self._make_one( + project=PROJECT, credentials=credentials, extra_headers=custom_headers + ) + self.assertEqual( + client._connection.API_BASE_URL, client._connection.DEFAULT_API_ENDPOINT + ) + self.assertEqual(client._connection.extra_headers, custom_headers) + def test_ctor_wo_project(self): PROJECT = "PROJECT" credentials = _make_credentials(project=PROJECT) diff --git a/tests/unit/test_transfer_manager.py b/tests/unit/test_transfer_manager.py index 54284becd..9042b05e0 100644 --- a/tests/unit/test_transfer_manager.py +++ b/tests/unit/test_transfer_manager.py @@ -850,6 +850,9 @@ def test_upload_chunks_concurrently_with_metadata_and_encryption(): custom_metadata = {"key_a": "value_a", "key_b": "value_b"} encryption_key = "b23ff11bba187db8c37077e6af3b25b8" kms_key_name = "sample_key_name" + custom_headers = { + "x-goog-custom-audit-foo": "bar", + } METADATA = { "cache_control": "private", @@ -862,7 +865,9 @@ def test_upload_chunks_concurrently_with_metadata_and_encryption(): bucket = mock.Mock() bucket.name = "bucket" - bucket.client = _PickleableMockClient(identify_as_client=True) + bucket.client = _PickleableMockClient( + identify_as_client=True, extra_headers=custom_headers + ) transport = bucket.client._http user_project = "my_project" bucket.user_project = user_project @@ -920,6 +925,7 @@ def test_upload_chunks_concurrently_with_metadata_and_encryption(): "x-goog-meta-key_b": "value_b", "x-goog-user-project": "my_project", "x-goog-encryption-kms-key-name": "sample_key_name", + **custom_headers, } container_cls_mock.assert_called_once_with( URL, FILENAME, headers=expected_headers @@ -966,10 +972,11 @@ def get_api_base_url_for_mtls(): class _PickleableMockClient: - def __init__(self, identify_as_client=False): + def __init__(self, identify_as_client=False, extra_headers={}): self._http = "my_transport" # used as an identifier for "called_with" self._connection = _PickleableMockConnection() self.identify_as_client = identify_as_client + self._extra_headers = extra_headers @property def __class__(self): @@ -1083,11 +1090,17 @@ def test__get_pool_class_and_requirements_error(): def test__reduce_client(): fake_cache = {} client = mock.Mock() + custom_headers = { + "x-goog-custom-audit-foo": "bar", + } + client._extra_headers = custom_headers with mock.patch( "google.cloud.storage.transfer_manager._cached_clients", new=fake_cache ), mock.patch("google.cloud.storage.transfer_manager.Client"): - transfer_manager._reduce_client(client) + replicated_client, kwargs = transfer_manager._reduce_client(client) + assert replicated_client is not None + assert custom_headers in kwargs def test__call_method_on_maybe_pickled_blob():