From 7aa960ba5cc81482e48a46d391d7453e49550d49 Mon Sep 17 00:00:00 2001 From: Chandra Sirimala Date: Tue, 11 Aug 2026 13:11:55 +0000 Subject: [PATCH 1/4] fix(storage): attach trailing checksum on final chunk of streaming resumable uploads (b/471191823) --- .../google/cloud/storage/_media/_upload.py | 5 ++ .../unit/requests/test_upload.py | 49 ++++++++++++++++ .../tests/system/test_blob.py | 58 +++++++++++++++++++ 3 files changed, 112 insertions(+) diff --git a/packages/google-cloud-storage/google/cloud/storage/_media/_upload.py b/packages/google-cloud-storage/google/cloud/storage/_media/_upload.py index acbc04e715c9..819e496b868f 100644 --- a/packages/google-cloud-storage/google/cloud/storage/_media/_upload.py +++ b/packages/google-cloud-storage/google/cloud/storage/_media/_upload.py @@ -676,6 +676,11 @@ def _prepare_request(self): msg = _STREAM_ERROR_TEMPLATE.format(start_byte, self.bytes_uploaded) raise ValueError(msg) + if self._total_bytes is None and not content_range.endswith("/*"): + total_str = content_range.split("/")[-1] + if total_str.isdigit(): + self._total_bytes = int(total_str) + self._update_checksum(start_byte, payload) headers = { diff --git a/packages/google-cloud-storage/tests/resumable_media/unit/requests/test_upload.py b/packages/google-cloud-storage/tests/resumable_media/unit/requests/test_upload.py index e99855570694..4cc88f348abc 100644 --- a/packages/google-cloud-storage/tests/resumable_media/unit/requests/test_upload.py +++ b/packages/google-cloud-storage/tests/resumable_media/unit/requests/test_upload.py @@ -361,6 +361,55 @@ def test_recover(self): timeout=EXPECTED_TIMEOUT, ) + def test_transmit_next_chunk_streaming_final_chunk_attaches_checksum(self): + import base64 + import google_crc32c + + data = b"Streaming payload that finishes in one chunk." + upload = upload_mod.ResumableUpload( + RESUMABLE_URL, ONE_MB, checksum="crc32c" + ) + upload._stream = io.BytesIO(data) + upload._content_type = BASIC_CONTENT + upload._total_bytes = None # Unknown initial size (streaming write) + upload._resumable_url = "http://test.invalid?upload_id=not-none" + + crc32c_int = google_crc32c.value(data) + crc32c_b64 = base64.b64encode(crc32c_int.to_bytes(4, "big")).decode("utf-8") + + transport = mock.Mock(spec=["request"]) + put_response = mock.Mock( + status_code=http.client.OK, + headers={}, + json=mock.Mock(return_value={"crc32c": crc32c_b64}), + ) + transport.request.return_value = put_response + upload.transmit_next_chunk(transport) + + assert upload._total_bytes == len(data) + called_args, called_kwargs = transport.request.call_args + headers = called_kwargs["headers"] + assert "x-goog-hash" in headers + assert headers["x-goog-hash"] == f"crc32c={crc32c_b64}" + + def test_transmit_next_chunk_streaming_no_checksum_requested(self): + data = b"Streaming payload without checksum." + upload = upload_mod.ResumableUpload( + RESUMABLE_URL, ONE_MB, checksum=None + ) + upload._stream = io.BytesIO(data) + upload._content_type = BASIC_CONTENT + upload._total_bytes = None + upload._resumable_url = "http://test.invalid?upload_id=not-none" + + transport = self._chunk_mock(http.client.OK, {}) + upload.transmit_next_chunk(transport) + + assert upload._total_bytes == len(data) + called_args, called_kwargs = transport.request.call_args + headers = called_kwargs["headers"] + assert "x-goog-hash" not in headers + def test_mpu_container(): container = upload_mod.XMLMPUContainer(EXAMPLE_XML_UPLOAD_URL, filename) diff --git a/packages/google-cloud-storage/tests/system/test_blob.py b/packages/google-cloud-storage/tests/system/test_blob.py index 8cea2add2d4a..8193728605ba 100644 --- a/packages/google-cloud-storage/tests/system/test_blob.py +++ b/packages/google-cloud-storage/tests/system/test_blob.py @@ -1345,3 +1345,61 @@ def test_blob_contexts_custom_setter(shared_bucket, blobs_to_delete): blob.reload() assert blob.contexts.custom["k1"].value == "v1-updated" assert blob.contexts.custom["k2"].value == "v2" + + +def test_upload_from_file_streaming_with_trailing_checksum_validation( + shared_bucket, blobs_to_delete +): + import io + import os + import google_crc32c + import base64 + + blob_name = f"StreamingTrailingChecksum-{uuid.uuid4().hex}" + blob = shared_bucket.blob(blob_name) + blobs_to_delete.append(blob) + + # Payload > 8 MiB to trigger chunked resumable upload + payload = os.urandom(8 * 1024 * 1024 + 1024) + io_stream = io.BytesIO(payload) + + # Calculate expected CRC32C base64 hash + expected_crc32c_int = google_crc32c.value(payload) + expected_crc32c_b64 = base64.b64encode( + expected_crc32c_int.to_bytes(4, "big") + ).decode("utf-8") + + # Upload from stream WITHOUT passing size (routes to resumable upload with total_bytes=None) + blob.upload_from_file(io_stream, checksum="crc32c") + + blob.reload() + assert blob.size == len(payload) + assert blob.crc32c == expected_crc32c_b64 + + +def test_upload_from_file_streaming_corrupted_checksum_rejection( + shared_bucket, blobs_to_delete +): + import io + import os + import pytest + from google.api_core.exceptions import BadRequest + from google.cloud.storage.exceptions import DataCorruption + + blob_name = f"StreamingCorruptedChecksum-{uuid.uuid4().hex}" + blob = shared_bucket.blob(blob_name) + blobs_to_delete.append(blob) + + payload = os.urandom(8 * 1024 * 1024 + 1024) + io_stream = io.BytesIO(payload) + + # Supply an intentionally corrupted/mismatched checksum + bad_crc32c_b64 = "AAAAAA==" + + with pytest.raises(BadRequest): + blob.upload_from_file( + io_stream, + checksum="crc32c", + crc32c_checksum_value=bad_crc32c_b64, + ) + From 229cf40f338a4e42cc409a9e55526226cc3032dc Mon Sep 17 00:00:00 2001 From: Chandra Sirimala Date: Tue, 11 Aug 2026 13:38:40 +0000 Subject: [PATCH 2/4] test(storage): update test_large_file_write_from_stream_w_failed_checksum to accept BadRequest when trailing checksum header is rejected by GCS --- packages/google-cloud-storage/tests/system/test_blob.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/google-cloud-storage/tests/system/test_blob.py b/packages/google-cloud-storage/tests/system/test_blob.py index 8193728605ba..7f481112901c 100644 --- a/packages/google-cloud-storage/tests/system/test_blob.py +++ b/packages/google-cloud-storage/tests/system/test_blob.py @@ -134,7 +134,7 @@ def test_large_file_write_from_stream_w_failed_checksum( "google.cloud.storage._media._helpers.prepare_checksum_digest", return_value="FFFFFF==", ): - with pytest.raises(DataCorruption): + with pytest.raises((DataCorruption, exceptions.BadRequest)): blob.upload_from_file(file_obj, checksum="crc32c") assert not blob.exists() From 73bb6c85ec776b2c29a90ee88a897c21cbfb1091 Mon Sep 17 00:00:00 2001 From: Chandra Sirimala Date: Tue, 11 Aug 2026 13:41:59 +0000 Subject: [PATCH 3/4] style(storage): fix lint errors (line length, unused import, newline) in unit and system tests --- .../tests/resumable_media/unit/requests/test_upload.py | 3 ++- packages/google-cloud-storage/tests/system/test_blob.py | 5 ++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/google-cloud-storage/tests/resumable_media/unit/requests/test_upload.py b/packages/google-cloud-storage/tests/resumable_media/unit/requests/test_upload.py index 4cc88f348abc..05edd6af48b2 100644 --- a/packages/google-cloud-storage/tests/resumable_media/unit/requests/test_upload.py +++ b/packages/google-cloud-storage/tests/resumable_media/unit/requests/test_upload.py @@ -375,7 +375,8 @@ def test_transmit_next_chunk_streaming_final_chunk_attaches_checksum(self): upload._resumable_url = "http://test.invalid?upload_id=not-none" crc32c_int = google_crc32c.value(data) - crc32c_b64 = base64.b64encode(crc32c_int.to_bytes(4, "big")).decode("utf-8") + crc32c_bytes = crc32c_int.to_bytes(4, "big") + crc32c_b64 = base64.b64encode(crc32c_bytes).decode("utf-8") transport = mock.Mock(spec=["request"]) put_response = mock.Mock( diff --git a/packages/google-cloud-storage/tests/system/test_blob.py b/packages/google-cloud-storage/tests/system/test_blob.py index 7f481112901c..f7041ad88813 100644 --- a/packages/google-cloud-storage/tests/system/test_blob.py +++ b/packages/google-cloud-storage/tests/system/test_blob.py @@ -1369,7 +1369,8 @@ def test_upload_from_file_streaming_with_trailing_checksum_validation( expected_crc32c_int.to_bytes(4, "big") ).decode("utf-8") - # Upload from stream WITHOUT passing size (routes to resumable upload with total_bytes=None) + # Upload from stream WITHOUT passing size parameter + # (routes to resumable upload with total_bytes=None) blob.upload_from_file(io_stream, checksum="crc32c") blob.reload() @@ -1384,7 +1385,6 @@ def test_upload_from_file_streaming_corrupted_checksum_rejection( import os import pytest from google.api_core.exceptions import BadRequest - from google.cloud.storage.exceptions import DataCorruption blob_name = f"StreamingCorruptedChecksum-{uuid.uuid4().hex}" blob = shared_bucket.blob(blob_name) @@ -1402,4 +1402,3 @@ def test_upload_from_file_streaming_corrupted_checksum_rejection( checksum="crc32c", crc32c_checksum_value=bad_crc32c_b64, ) - From c4783170e30667c89b38b7f368bb19721038b800 Mon Sep 17 00:00:00 2001 From: Chandra Sirimala Date: Tue, 11 Aug 2026 14:47:06 +0000 Subject: [PATCH 4/4] style(storage): format and sort imports with ruff to satisfy CI lint checks --- .../resumable_media/unit/requests/test_upload.py | 12 ++++-------- .../google-cloud-storage/tests/system/test_blob.py | 5 +++-- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/packages/google-cloud-storage/tests/resumable_media/unit/requests/test_upload.py b/packages/google-cloud-storage/tests/resumable_media/unit/requests/test_upload.py index 05edd6af48b2..fcdbe38c1b4c 100644 --- a/packages/google-cloud-storage/tests/resumable_media/unit/requests/test_upload.py +++ b/packages/google-cloud-storage/tests/resumable_media/unit/requests/test_upload.py @@ -18,9 +18,8 @@ import tempfile from unittest import mock -import pytest # type: ignore - import google.cloud.storage._media.requests.upload as upload_mod +import pytest # type: ignore URL_PREFIX = "https://www.googleapis.com/upload/storage/v1/b/{BUCKET}/o" SIMPLE_URL = URL_PREFIX + "?uploadType=media&name={OBJECT}" @@ -363,12 +362,11 @@ def test_recover(self): def test_transmit_next_chunk_streaming_final_chunk_attaches_checksum(self): import base64 + import google_crc32c data = b"Streaming payload that finishes in one chunk." - upload = upload_mod.ResumableUpload( - RESUMABLE_URL, ONE_MB, checksum="crc32c" - ) + upload = upload_mod.ResumableUpload(RESUMABLE_URL, ONE_MB, checksum="crc32c") upload._stream = io.BytesIO(data) upload._content_type = BASIC_CONTENT upload._total_bytes = None # Unknown initial size (streaming write) @@ -395,9 +393,7 @@ def test_transmit_next_chunk_streaming_final_chunk_attaches_checksum(self): def test_transmit_next_chunk_streaming_no_checksum_requested(self): data = b"Streaming payload without checksum." - upload = upload_mod.ResumableUpload( - RESUMABLE_URL, ONE_MB, checksum=None - ) + upload = upload_mod.ResumableUpload(RESUMABLE_URL, ONE_MB, checksum=None) upload._stream = io.BytesIO(data) upload._content_type = BASIC_CONTENT upload._total_bytes = None diff --git a/packages/google-cloud-storage/tests/system/test_blob.py b/packages/google-cloud-storage/tests/system/test_blob.py index f7041ad88813..76b93d2789b0 100644 --- a/packages/google-cloud-storage/tests/system/test_blob.py +++ b/packages/google-cloud-storage/tests/system/test_blob.py @@ -23,7 +23,6 @@ import mock import pytest from google.api_core import exceptions - from google.cloud.storage._helpers import _base64_md5hash from google.cloud.storage.exceptions import DataCorruption @@ -1350,10 +1349,11 @@ def test_blob_contexts_custom_setter(shared_bucket, blobs_to_delete): def test_upload_from_file_streaming_with_trailing_checksum_validation( shared_bucket, blobs_to_delete ): + import base64 import io import os + import google_crc32c - import base64 blob_name = f"StreamingTrailingChecksum-{uuid.uuid4().hex}" blob = shared_bucket.blob(blob_name) @@ -1383,6 +1383,7 @@ def test_upload_from_file_streaming_corrupted_checksum_rejection( ): import io import os + import pytest from google.api_core.exceptions import BadRequest