Skip to content
This repository was archived by the owner on Mar 31, 2026. It is now read-only.

Commit a7ecd85

Browse files
andrewsgandrewsg
committed
docs: Add samples to multithread branch (#918)
* add samples, tests pending * add snippet tests * snippet and snippets_test.py linting * snippets; recursive directory creation; rename some params * Add directory upload snippet
1 parent e9ca76d commit a7ecd85

File tree

4 files changed

+443
-67
lines changed

4 files changed

+443
-67
lines changed

‎google/cloud/storage/transfer_manager.py‎

Lines changed: 65 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,17 @@
1616

1717
import concurrent.futures
1818

19+
import os
1920
import tempfile
21+
import warnings
2022

2123
from google.api_core import exceptions
2224

25+
warnings.warn(
26+
"The module `transfer_manager` is a preview feature. Functionality and API "
27+
"may change. This warning will be removed in a future release."
28+
)
29+
2330

2431
DEFAULT_CHUNK_SIZE = 200 * 1024 * 1024
2532

@@ -305,7 +312,7 @@ def download_range_via_tempfile(blob, start, end, download_kwargs):
305312
def upload_many_from_filenames(
306313
bucket,
307314
filenames,
308-
root="",
315+
source_directory="",
309316
blob_name_prefix="",
310317
skip_if_exists=False,
311318
blob_constructor_kwargs=None,
@@ -321,10 +328,10 @@ def upload_many_from_filenames(
321328
The destination blobs are automatically created, with blob names based on
322329
the source filenames and the blob_name_prefix.
323330
324-
For example, if the `filenames` include "images/icon.jpg", `root` is
325-
"/home/myuser/", and `blob_name_prefix` is "myfiles/", then the file at
326-
"/home/myuser/images/icon.jpg" will be uploaded to a blob named
327-
"myfiles/images/icon.jpg".
331+
For example, if the `filenames` include "images/icon.jpg",
332+
`source_directory` is "/home/myuser/", and `blob_name_prefix` is "myfiles/",
333+
then the file at "/home/myuser/images/icon.jpg" will be uploaded to a blob
334+
named "myfiles/images/icon.jpg".
328335
329336
:type bucket: 'google.cloud.storage.bucket.Bucket'
330337
:param bucket:
@@ -333,35 +340,35 @@ def upload_many_from_filenames(
333340
:type filenames: list(str)
334341
:param filenames:
335342
A list of filenames to be uploaded. This may include part of the path.
336-
The full path to the file must be root + filename. The filename is
337-
separate from the root because the filename will also determine the
338-
name of the destination blob.
343+
The full path to the file must be source_directory + filename.
339344
340-
:type root: str
341-
:param root:
342-
A string that will be prepended to each filename in the input list, in
343-
order to find the source file for each blob. Unlike the filename itself,
344-
the root string does not affect the name of the uploaded blob itself.
345-
The root string will usually end in "/" (or "\\" depending on platform)
346-
but is not required to do so.
345+
:type source_directory: str
346+
:param source_directory:
347+
A string that will be prepended (with os.path.join()) to each filename
348+
in the input list, in order to find the source file for each blob.
349+
Unlike the filename itself, the source_directory does not affect the
350+
name of the uploaded blob.
347351
348-
For instance, if the root string is "/tmp/img-" and a filename is
352+
For instance, if the source_directory is "/tmp/img/" and a filename is
349353
"0001.jpg", with an empty blob_name_prefix, then the file uploaded will
350-
be "/tmp/img-0001.jpg" and the destination blob will be "0001.jpg".
354+
be "/tmp/img/0001.jpg" and the destination blob will be "0001.jpg".
351355
352356
This parameter can be an empty string.
353357
358+
Note that this parameter allows directory traversal (e.g. "/", "../")
359+
and is not intended for unsanitized end user input.
360+
354361
:type blob_name_prefix: str
355362
:param blob_name_prefix:
356363
A string that will be prepended to each filename in the input list, in
357364
order to determine the name of the destination blob. Unlike the filename
358365
itself, the prefix string does not affect the location the library will
359366
look for the source data on the local filesystem.
360367
361-
For instance, if the root is "/tmp/img-", the blob_name_prefix is
362-
"myuser/mystuff-" and a filename is "0001.jpg" then the file uploaded
363-
will be "/tmp/img-0001.jpg" and the destination blob will be
364-
"myuser/mystuff-0001.jpg".
368+
For instance, if the source_directory is "/tmp/img/", the
369+
blob_name_prefix is "myuser/mystuff-" and a filename is "0001.jpg" then
370+
the file uploaded will be "/tmp/img/0001.jpg" and the destination blob
371+
will be "myuser/mystuff-0001.jpg".
365372
366373
The blob_name_prefix can be blank (an empty string).
367374
@@ -370,7 +377,7 @@ def upload_many_from_filenames(
370377
If True, blobs that already have a live version will not be overwritten.
371378
This is accomplished by setting "if_generation_match = 0" on uploads.
372379
Uploads so skipped will result in a 412 Precondition Failed response
373-
code, which will be included in the return value but not raised
380+
code, which will be included in the return value, but not raised
374381
as an exception regardless of the value of raise_exception.
375382
376383
:type blob_constructor_kwargs: dict
@@ -426,7 +433,7 @@ def upload_many_from_filenames(
426433
file_blob_pairs = []
427434

428435
for filename in filenames:
429-
path = root + filename
436+
path = os.path.join(source_directory, filename)
430437
blob_name = blob_name_prefix + filename
431438
blob = bucket.blob(blob_name, **blob_constructor_kwargs)
432439
file_blob_pairs.append((path, blob))
@@ -444,26 +451,27 @@ def upload_many_from_filenames(
444451
def download_many_to_path(
445452
bucket,
446453
blob_names,
447-
path_root="",
454+
destination_directory="",
448455
blob_name_prefix="",
449456
download_kwargs=None,
450457
max_workers=None,
451458
deadline=None,
459+
create_directories=True,
452460
raise_exception=False,
453461
):
454462
"""Download many files concurrently by their blob names.
455463
456464
This function is a PREVIEW FEATURE: the API may change in a future version.
457465
458-
The destination files are automatically created, with filenames based on
459-
the source blob_names and the path_root.
466+
The destination files are automatically created, with paths based on the
467+
source blob_names and the destination_directory.
460468
461469
The destination files are not automatically deleted if their downloads fail,
462470
so please check the return value of this function for any exceptions, or
463471
enable `raise_exception=True`, and process the files accordingly.
464472
465-
For example, if the `blob_names` include "icon.jpg", `path_root` is
466-
"/home/myuser/", and `blob_name_prefix` is "images/", then the blob named
473+
For example, if the `blob_names` include "icon.jpg", `destination_directory`
474+
is "/home/myuser/", and `blob_name_prefix` is "images/", then the blob named
467475
"images/icon.jpg" will be downloaded to a file named
468476
"/home/myuser/icon.jpg".
469477
@@ -482,26 +490,31 @@ def download_many_to_path(
482490
the blob names that need not be part of the destination path should be
483491
included in the blob_name_prefix.
484492
485-
:type path_root: str
486-
:param path_root:
487-
A string that will be prepended to each blob_name in the input list,
488-
in order to determine the destination path for that blob. The path_root
489-
string will usually end in "/" (or "\\" depending on platform) but is
490-
not required to do so. For instance, if the path_root string is
491-
"/tmp/img-" and a blob_name is "0001.jpg", with an empty
492-
blob_name_prefix, then the source blob "0001.jpg" will be downloaded to
493-
destination "/tmp/img-0001.jpg" . This parameter can be an empty string.
493+
:type destination_directory: str
494+
:param destination_directory:
495+
A string that will be prepended (with os.path.join()) to each blob_name
496+
in the input list, in order to determine the destination path for that
497+
blob.
498+
499+
For instance, if the destination_directory string is "/tmp/img" and a
500+
blob_name is "0001.jpg", with an empty blob_name_prefix, then the source
501+
blob "0001.jpg" will be downloaded to destination "/tmp/img/0001.jpg" .
502+
503+
This parameter can be an empty string.
504+
505+
Note that this parameter allows directory traversal (e.g. "/", "../")
506+
and is not intended for unsanitized end user input.
494507
495508
:type blob_name_prefix: str
496509
:param blob_name_prefix:
497510
A string that will be prepended to each blob_name in the input list, in
498511
order to determine the name of the source blob. Unlike the blob_name
499512
itself, the prefix string does not affect the destination path on the
500-
local filesystem. For instance, if the path_root is "/tmp/img-", the
501-
blob_name_prefix is "myuser/mystuff-" and a blob_name is "0001.jpg" then
502-
the source blob "myuser/mystuff-0001.jpg" will be downloaded to
503-
"/tmp/img-0001.jpg". The blob_name_prefix can be blank (an empty
504-
string).
513+
local filesystem. For instance, if the destination_directory is
514+
"/tmp/img/", the blob_name_prefix is "myuser/mystuff-" and a blob_name
515+
is "0001.jpg" then the source blob "myuser/mystuff-0001.jpg" will be
516+
downloaded to "/tmp/img/0001.jpg". The blob_name_prefix can be blank
517+
(an empty string).
505518
506519
:type download_kwargs: dict
507520
:param download_kwargs:
@@ -523,6 +536,12 @@ def download_many_to_path(
523536
progress and concurrent.futures.TimeoutError will be raised. This can be
524537
left as the default of None (no deadline) for most use cases.
525538
539+
:type create_directories: bool
540+
:param create_directories:
541+
If True, recursively create any directories that do not exist. For
542+
instance, if downloading object "images/img001.png", create the
543+
directory "images" before downloading.
544+
526545
:type raise_exception: bool
527546
:param raise_exception:
528547
If True, instead of adding exceptions to the list of return values,
@@ -545,7 +564,10 @@ def download_many_to_path(
545564

546565
for blob_name in blob_names:
547566
full_blob_name = blob_name_prefix + blob_name
548-
path = path_root + blob_name
567+
path = os.path.join(destination_directory, blob_name)
568+
if create_directories:
569+
directory, _ = os.path.split(path)
570+
os.makedirs(directory, exist_ok=True)
549571
blob_file_pairs.append((bucket.blob(full_blob_name), path))
550572

551573
return download_many(

0 commit comments

Comments
 (0)