Skip to content

Commit 6279b2a

Browse files
cojencocojenco
authored
samples: add generation-match preconditions to selected samples (#949)
* samples: add preconditions to objects.delete * add preconditons to rewrite category samples * add compose and update previous changes * preconditions to rewrites and encrypted uploads * add preconditions to objects insert * refine optional block wording and flow * update test
1 parent 9fd1d76 commit 6279b2a

13 files changed

Lines changed: 134 additions & 31 deletions

‎packages/google-cloud-storage/samples/snippets/encryption_test.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,15 +47,18 @@ def test_generate_encryption_key(capsys):
4747

4848

4949
def test_upload_encrypted_blob():
50+
blob_name = f"test_upload_encrypted_{uuid.uuid4().hex}"
5051
with tempfile.NamedTemporaryFile() as source_file:
5152
source_file.write(b"test")
5253

5354
storage_upload_encrypted_file.upload_encrypted_blob(
5455
BUCKET,
5556
source_file.name,
56-
"test_encrypted_upload_blob",
57+
blob_name,
5758
TEST_ENCRYPTION_KEY,
5859
)
60+
bucket = storage.Client().bucket(BUCKET)
61+
bucket.delete_blob(blob_name)
5962

6063

6164
@pytest.fixture(scope="module")

‎packages/google-cloud-storage/samples/snippets/snippets_test.py‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -235,14 +235,16 @@ def test_upload_blob_from_stream(test_bucket, capsys):
235235

236236

237237
def test_upload_blob_with_kms(test_bucket):
238+
blob_name = f"test_upload_with_kms_{uuid.uuid4().hex}"
238239
with tempfile.NamedTemporaryFile() as source_file:
239240
source_file.write(b"test")
240241
storage_upload_with_kms_key.upload_blob_with_kms(
241-
test_bucket.name, source_file.name, "test_upload_blob_encrypted", KMS_KEY
242+
test_bucket.name, source_file.name, blob_name, KMS_KEY,
242243
)
243244
bucket = storage.Client().bucket(test_bucket.name)
244-
kms_blob = bucket.get_blob("test_upload_blob_encrypted")
245+
kms_blob = bucket.get_blob(blob_name)
245246
assert kms_blob.kms_key_name.startswith(KMS_KEY)
247+
test_bucket.delete_blob(blob_name)
246248

247249

248250
def test_async_upload(bucket, capsys):
@@ -390,7 +392,7 @@ def test_move_blob(test_bucket_create, test_blob):
390392
print(f"test_move_blob not found in bucket {test_bucket_create.name}")
391393

392394
storage_move_file.move_blob(
393-
bucket.name, test_blob.name, test_bucket_create.name, "test_move_blob"
395+
bucket.name, test_blob.name, test_bucket_create.name, "test_move_blob",
394396
)
395397

396398
assert test_bucket_create.get_blob("test_move_blob") is not None
@@ -406,7 +408,7 @@ def test_copy_blob(test_blob):
406408
pass
407409

408410
storage_copy_file.copy_blob(
409-
bucket.name, test_blob.name, bucket.name, "test_copy_blob"
411+
bucket.name, test_blob.name, bucket.name, "test_copy_blob",
410412
)
411413

412414
assert bucket.get_blob("test_copy_blob") is not None
@@ -545,7 +547,7 @@ def test_define_bucket_website_configuration(test_bucket):
545547
def test_object_get_kms_key(test_bucket):
546548
with tempfile.NamedTemporaryFile() as source_file:
547549
storage_upload_with_kms_key.upload_blob_with_kms(
548-
test_bucket.name, source_file.name, "test_upload_blob_encrypted", KMS_KEY
550+
test_bucket.name, source_file.name, "test_upload_blob_encrypted", KMS_KEY,
549551
)
550552
kms_key = storage_object_get_kms_key.object_get_kms_key(
551553
test_bucket.name, "test_upload_blob_encrypted"
@@ -562,7 +564,7 @@ def test_storage_compose_file(test_bucket):
562564

563565
with tempfile.NamedTemporaryFile() as dest_file:
564566
destination = storage_compose_file.compose_file(
565-
test_bucket.name, source_files[0], source_files[1], dest_file.name
567+
test_bucket.name, source_files[0], source_files[1], dest_file.name,
566568
)
567569
composed = destination.download_as_string()
568570

@@ -602,7 +604,7 @@ def test_change_default_storage_class(test_bucket, capsys):
602604

603605
def test_change_file_storage_class(test_blob, capsys):
604606
blob = storage_change_file_storage_class.change_file_storage_class(
605-
test_blob.bucket.name, test_blob.name
607+
test_blob.bucket.name, test_blob.name,
606608
)
607609
out, _ = capsys.readouterr()
608610
assert f"Blob {blob.name} in bucket {blob.bucket.name}" in out

‎packages/google-cloud-storage/samples/snippets/storage_change_file_storage_class.py‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,9 +27,17 @@ def change_file_storage_class(bucket_name, blob_name):
2727

2828
storage_client = storage.Client()
2929

30-
bucket = storage_client.get_bucket(bucket_name)
31-
blob = bucket.get_blob(blob_name)
32-
blob.update_storage_class("NEARLINE")
30+
bucket = storage_client.bucket(bucket_name)
31+
blob = bucket.blob(blob_name)
32+
generation_match_precondition = None
33+
34+
# Optional: set a generation-match precondition to avoid potential race
35+
# conditions and data corruptions. The request is aborted if the
36+
# object's generation number does not match your precondition.
37+
blob.reload() # Fetch blob metadata to use in generation_match_precondition.
38+
generation_match_precondition = blob.generation
39+
40+
blob.update_storage_class("NEARLINE", if_generation_match=generation_match_precondition)
3341

3442
print(
3543
"Blob {} in bucket {} had its storage class set to {}".format(

‎packages/google-cloud-storage/samples/snippets/storage_compose_file.py‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -32,9 +32,19 @@ def compose_file(bucket_name, first_blob_name, second_blob_name, destination_blo
3232
destination = bucket.blob(destination_blob_name)
3333
destination.content_type = "text/plain"
3434

35-
# sources is a list of Blob instances, up to the max of 32 instances per request
36-
sources = [bucket.get_blob(first_blob_name), bucket.get_blob(second_blob_name)]
37-
destination.compose(sources)
35+
# Note sources is a list of Blob instances, up to the max of 32 instances per request
36+
sources = [bucket.blob(first_blob_name), bucket.blob(second_blob_name)]
37+
38+
# Optional: set a generation-match precondition to avoid potential race conditions
39+
# and data corruptions. The request to compose is aborted if the object's
40+
# generation number does not match your precondition. For a destination
41+
# object that does not yet exist, set the if_generation_match precondition to 0.
42+
# If the destination object already exists in your bucket, set instead a
43+
# generation-match precondition using its generation number.
44+
# There is also an `if_source_generation_match` parameter, which is not used in this example.
45+
destination_generation_match_precondition = 0
46+
47+
destination.compose(sources, if_generation_match=destination_generation_match_precondition)
3848

3949
print(
4050
"New composite object {} in the bucket {} was created by combining {} and {}".format(

‎packages/google-cloud-storage/samples/snippets/storage_copy_file.py‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121

2222

2323
def copy_blob(
24-
bucket_name, blob_name, destination_bucket_name, destination_blob_name
24+
bucket_name, blob_name, destination_bucket_name, destination_blob_name,
2525
):
2626
"""Copies a blob from one bucket to another with a new name."""
2727
# bucket_name = "your-bucket-name"
@@ -35,8 +35,17 @@ def copy_blob(
3535
source_blob = source_bucket.blob(blob_name)
3636
destination_bucket = storage_client.bucket(destination_bucket_name)
3737

38+
# Optional: set a generation-match precondition to avoid potential race conditions
39+
# and data corruptions. The request to copy is aborted if the object's
40+
# generation number does not match your precondition. For a destination
41+
# object that does not yet exist, set the if_generation_match precondition to 0.
42+
# If the destination object already exists in your bucket, set instead a
43+
# generation-match precondition using its generation number.
44+
# There is also an `if_source_generation_match` parameter, which is not used in this example.
45+
destination_generation_match_precondition = 0
46+
3847
blob_copy = source_bucket.copy_blob(
39-
source_blob, destination_bucket, destination_blob_name
48+
source_blob, destination_bucket, destination_blob_name, if_generation_match=destination_generation_match_precondition,
4049
)
4150

4251
print(

‎packages/google-cloud-storage/samples/snippets/storage_copy_file_archived_generation.py‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,13 +36,22 @@ def copy_file_archived_generation(
3636
source_blob = source_bucket.blob(blob_name)
3737
destination_bucket = storage_client.bucket(destination_bucket_name)
3838

39+
# Optional: set a generation-match precondition to avoid potential race conditions
40+
# and data corruptions. The request to copy is aborted if the object's
41+
# generation number does not match your precondition. For a destination
42+
# object that does not yet exist, set the if_generation_match precondition to 0.
43+
# If the destination object already exists in your bucket, set instead a
44+
# generation-match precondition using its generation number.
45+
destination_generation_match_precondition = 0
46+
47+
# source_generation selects a specific revision of the source object, as opposed to the latest version.
3948
blob_copy = source_bucket.copy_blob(
40-
source_blob, destination_bucket, destination_blob_name, source_generation=generation
49+
source_blob, destination_bucket, destination_blob_name, source_generation=generation, if_generation_match=destination_generation_match_precondition
4150
)
4251

4352
print(
4453
"Generation {} of the blob {} in bucket {} copied to blob {} in bucket {}.".format(
45-
source_blob.generation,
54+
generation,
4655
source_blob.name,
4756
source_bucket.name,
4857
blob_copy.name,

‎packages/google-cloud-storage/samples/snippets/storage_delete_file.py‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,15 @@ def delete_blob(bucket_name, blob_name):
2929

3030
bucket = storage_client.bucket(bucket_name)
3131
blob = bucket.blob(blob_name)
32-
blob.delete()
32+
generation_match_precondition = None
33+
34+
# Optional: set a generation-match precondition to avoid potential race conditions
35+
# and data corruptions. The request to delete is aborted if the object's
36+
# generation number does not match your precondition.
37+
blob.reload() # Fetch blob metadata to use in generation_match_precondition.
38+
generation_match_precondition = blob.generation
39+
40+
blob.delete(if_generation_match=generation_match_precondition)
3341

3442
print(f"Blob {blob_name} deleted.")
3543

‎packages/google-cloud-storage/samples/snippets/storage_move_file.py‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
from google.cloud import storage
2121

2222

23-
def move_blob(bucket_name, blob_name, destination_bucket_name, destination_blob_name):
23+
def move_blob(bucket_name, blob_name, destination_bucket_name, destination_blob_name,):
2424
"""Moves a blob from one bucket to another with a new name."""
2525
# The ID of your GCS bucket
2626
# bucket_name = "your-bucket-name"
@@ -37,8 +37,17 @@ def move_blob(bucket_name, blob_name, destination_bucket_name, destination_blob_
3737
source_blob = source_bucket.blob(blob_name)
3838
destination_bucket = storage_client.bucket(destination_bucket_name)
3939

40+
# Optional: set a generation-match precondition to avoid potential race conditions
41+
# and data corruptions. The request is aborted if the object's
42+
# generation number does not match your precondition. For a destination
43+
# object that does not yet exist, set the if_generation_match precondition to 0.
44+
# If the destination object already exists in your bucket, set instead a
45+
# generation-match precondition using its generation number.
46+
# There is also an `if_source_generation_match` parameter, which is not used in this example.
47+
destination_generation_match_precondition = 0
48+
4049
blob_copy = source_bucket.copy_blob(
41-
source_blob, destination_bucket, destination_blob_name
50+
source_blob, destination_bucket, destination_blob_name, if_generation_match=destination_generation_match_precondition,
4251
)
4352
source_bucket.delete_blob(blob_name)
4453

‎packages/google-cloud-storage/samples/snippets/storage_object_csek_to_cmek.py‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -33,12 +33,22 @@ def object_csek_to_cmek(bucket_name, blob_name, encryption_key, kms_key_name):
3333

3434
current_encryption_key = base64.b64decode(encryption_key)
3535
source_blob = bucket.blob(blob_name, encryption_key=current_encryption_key)
36-
3736
destination_blob = bucket.blob(blob_name, kms_key_name=kms_key_name)
38-
token, rewritten, total = destination_blob.rewrite(source_blob)
37+
generation_match_precondition = None
38+
token = None
39+
40+
# Optional: set a generation-match precondition to avoid potential race conditions
41+
# and data corruptions. The request to rewrite is aborted if the object's
42+
# generation number does not match your precondition.
43+
source_blob.reload() # Fetch blob metadata to use in generation_match_precondition.
44+
generation_match_precondition = source_blob.generation
3945

40-
while token is not None:
41-
token, rewritten, total = destination_blob.rewrite(source_blob, token=token)
46+
while True:
47+
token, bytes_rewritten, total_bytes = destination_blob.rewrite(
48+
source_blob, token=token, if_generation_match=generation_match_precondition
49+
)
50+
if token is None:
51+
break
4252

4353
print(
4454
"Blob {} in bucket {} is now managed by the KMS key {} instead of a customer-supplied encryption key".format(

‎packages/google-cloud-storage/samples/snippets/storage_rotate_encryption_key.py‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,12 +42,18 @@ def rotate_encryption_key(
4242
destination_blob = bucket.blob(
4343
blob_name, encryption_key=new_encryption_key
4444
)
45-
45+
generation_match_precondition = None
4646
token = None
4747

48+
# Optional: set a generation-match precondition to avoid potential race conditions
49+
# and data corruptions. The request to rewrite is aborted if the object's
50+
# generation number does not match your precondition.
51+
source_blob.reload() # Fetch blob metadata to use in generation_match_precondition.
52+
generation_match_precondition = source_blob.generation
53+
4854
while True:
4955
token, bytes_rewritten, total_bytes = destination_blob.rewrite(
50-
source_blob, token=token
56+
source_blob, token=token, if_generation_match=generation_match_precondition
5157
)
5258
if token is None:
5359
break

0 commit comments

Comments
 (0)