Skip to content
This repository was archived by the owner on Mar 31, 2026. It is now read-only.

Commit d84c0dd

Browse files
andrewsgfrankyn
andrewsg
and
frankyn
authored
feat: add offset and includeTrailingPrefix options to list_blobs (#125)
* feat: add offset and includeTrailingPrefix options to list_blobs * lint * fix comment typo in system tests Co-authored-by: Frank Natividad <frankyn@users.noreply.github.com>
1 parent b6ad219 commit d84c0dd

5 files changed

Lines changed: 121 additions & 3 deletions

File tree

‎google/cloud/storage/bucket.py‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -894,6 +894,9 @@ def list_blobs(
894894
page_token=None,
895895
prefix=None,
896896
delimiter=None,
897+
start_offset=None,
898+
end_offset=None,
899+
include_trailing_delimiter=None,
897900
versions=None,
898901
projection="noAcl",
899902
fields=None,
@@ -926,6 +929,26 @@ def list_blobs(
926929
:param delimiter: (Optional) Delimiter, used with ``prefix`` to
927930
emulate hierarchy.
928931
932+
:type start_offset: str
933+
:param start_offset:
934+
(Optional) Filter results to objects whose names are
935+
lexicographically equal to or after ``startOffset``. If
936+
``endOffset`` is also set, the objects listed will have names
937+
between ``startOffset`` (inclusive) and ``endOffset`` (exclusive).
938+
939+
:type end_offset: str
940+
:param end_offset:
941+
(Optional) Filter results to objects whose names are
942+
lexicographically before ``endOffset``. If ``startOffset`` is also
943+
set, the objects listed will have names between ``startOffset``
944+
(inclusive) and ``endOffset`` (exclusive).
945+
946+
:type include_trailing_delimiter: boolean
947+
:param include_trailing_delimiter:
948+
(Optional) If true, objects that end in exactly one instance of
949+
``delimiter`` will have their metadata included in ``items`` in
950+
addition to ``prefixes``.
951+
929952
:type versions: bool
930953
:param versions: (Optional) Whether object versions should be returned
931954
as separate blobs.
@@ -967,6 +990,15 @@ def list_blobs(
967990
if delimiter is not None:
968991
extra_params["delimiter"] = delimiter
969992

993+
if start_offset is not None:
994+
extra_params["startOffset"] = start_offset
995+
996+
if end_offset is not None:
997+
extra_params["endOffset"] = end_offset
998+
999+
if include_trailing_delimiter is not None:
1000+
extra_params["includeTrailingDelimiter"] = include_trailing_delimiter
1001+
9701002
if versions is not None:
9711003
extra_params["versions"] = versions
9721004

‎google/cloud/storage/client.py‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -540,6 +540,9 @@ def list_blobs(
540540
page_token=None,
541541
prefix=None,
542542
delimiter=None,
543+
start_offset=None,
544+
end_offset=None,
545+
include_trailing_delimiter=None,
543546
versions=None,
544547
projection="noAcl",
545548
fields=None,
@@ -573,6 +576,24 @@ def list_blobs(
573576
(Optional) Delimiter, used with ``prefix`` to
574577
emulate hierarchy.
575578
579+
start_offset (str):
580+
(Optional) Filter results to objects whose names are
581+
lexicographically equal to or after ``startOffset``. If
582+
``endOffset`` is also set, the objects listed will have names
583+
between ``startOffset`` (inclusive) and ``endOffset``
584+
(exclusive).
585+
586+
end_offset (str):
587+
(Optional) Filter results to objects whose names are
588+
lexicographically before ``endOffset``. If ``startOffset`` is
589+
also set, the objects listed will have names between
590+
``startOffset`` (inclusive) and ``endOffset`` (exclusive).
591+
592+
include_trailing_delimiter (boolean):
593+
(Optional) If true, objects that end in exactly one instance of
594+
``delimiter`` will have their metadata included in ``items`` in
595+
addition to ``prefixes``.
596+
576597
versions (bool):
577598
(Optional) Whether object versions should be returned
578599
as separate blobs.
@@ -606,6 +627,9 @@ def list_blobs(
606627
page_token=page_token,
607628
prefix=prefix,
608629
delimiter=delimiter,
630+
start_offset=start_offset,
631+
end_offset=end_offset,
632+
include_trailing_delimiter=include_trailing_delimiter,
609633
versions=versions,
610634
projection=projection,
611635
fields=fields,

‎tests/system/test_system.py‎

Lines changed: 47 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -761,7 +761,7 @@ def test_fetch_object_and_check_content(self):
761761

762762
class TestStorageListFiles(TestStorageFiles):
763763

764-
FILENAMES = ("CloudLogo1", "CloudLogo2", "CloudLogo3")
764+
FILENAMES = ("CloudLogo1", "CloudLogo2", "CloudLogo3", "CloudLogo4")
765765

766766
@classmethod
767767
def setUpClass(cls):
@@ -818,18 +818,49 @@ def test_paginate_files(self):
818818
# Technically the iterator is exhausted.
819819
self.assertEqual(iterator.num_results, iterator.max_results)
820820
# But we modify the iterator to continue paging after
821-
# articially stopping after ``count`` items.
821+
# artificially stopping after ``count`` items.
822822
iterator.max_results = None
823823

824824
page2 = six.next(page_iter)
825825
last_blobs = list(page2)
826826
self.assertEqual(len(last_blobs), truncation_size)
827827

828+
@RetryErrors(unittest.TestCase.failureException)
829+
def test_paginate_files_with_offset(self):
830+
truncation_size = 1
831+
inclusive_start_offset = self.FILENAMES[1]
832+
exclusive_end_offset = self.FILENAMES[-1]
833+
desired_files = self.FILENAMES[1:-1]
834+
count = len(desired_files) - truncation_size
835+
iterator = self.bucket.list_blobs(
836+
max_results=count,
837+
start_offset=inclusive_start_offset,
838+
end_offset=exclusive_end_offset,
839+
)
840+
page_iter = iterator.pages
841+
842+
page1 = six.next(page_iter)
843+
blobs = list(page1)
844+
self.assertEqual(len(blobs), count)
845+
self.assertEqual(blobs[0].name, desired_files[0])
846+
self.assertIsNotNone(iterator.next_page_token)
847+
# Technically the iterator is exhausted.
848+
self.assertEqual(iterator.num_results, iterator.max_results)
849+
# But we modify the iterator to continue paging after
850+
# artificially stopping after ``count`` items.
851+
iterator.max_results = None
852+
853+
page2 = six.next(page_iter)
854+
last_blobs = list(page2)
855+
self.assertEqual(len(last_blobs), truncation_size)
856+
self.assertEqual(last_blobs[-1].name, desired_files[-1])
857+
828858

829859
class TestStoragePseudoHierarchy(TestStorageFiles):
830860

831861
FILENAMES = (
832862
"file01.txt",
863+
"parent/",
833864
"parent/file11.txt",
834865
"parent/child/file21.txt",
835866
"parent/child/file22.txt",
@@ -877,7 +908,9 @@ def test_first_level(self):
877908
iterator = self.bucket.list_blobs(delimiter="/", prefix="parent/")
878909
page = six.next(iterator.pages)
879910
blobs = list(page)
880-
self.assertEqual([blob.name for blob in blobs], ["parent/file11.txt"])
911+
self.assertEqual(
912+
[blob.name for blob in blobs], ["parent/", "parent/file11.txt"]
913+
)
881914
self.assertIsNone(iterator.next_page_token)
882915
self.assertEqual(iterator.prefixes, set(["parent/child/"]))
883916

@@ -909,6 +942,17 @@ def test_third_level(self):
909942
self.assertIsNone(iterator.next_page_token)
910943
self.assertEqual(iterator.prefixes, set())
911944

945+
@RetryErrors(unittest.TestCase.failureException)
946+
def test_include_trailing_delimiter(self):
947+
iterator = self.bucket.list_blobs(
948+
delimiter="/", include_trailing_delimiter=True
949+
)
950+
page = six.next(iterator.pages)
951+
blobs = list(page)
952+
self.assertEqual([blob.name for blob in blobs], ["file01.txt", "parent/"])
953+
self.assertIsNone(iterator.next_page_token)
954+
self.assertEqual(iterator.prefixes, set(["parent/"]))
955+
912956

913957
class TestStorageSignURLs(unittest.TestCase):
914958
BLOB_CONTENT = b"This time for sure, Rocky!"

‎tests/unit/test_bucket.py‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -733,6 +733,9 @@ def test_list_blobs_w_all_arguments_and_user_project(self):
733733
PAGE_TOKEN = "ABCD"
734734
PREFIX = "subfolder"
735735
DELIMITER = "/"
736+
START_OFFSET = "c"
737+
END_OFFSET = "g"
738+
INCLUDE_TRAILING_DELIMITER = True
736739
VERSIONS = True
737740
PROJECTION = "full"
738741
FIELDS = "items/contentLanguage,nextPageToken"
@@ -741,6 +744,9 @@ def test_list_blobs_w_all_arguments_and_user_project(self):
741744
"pageToken": PAGE_TOKEN,
742745
"prefix": PREFIX,
743746
"delimiter": DELIMITER,
747+
"startOffset": START_OFFSET,
748+
"endOffset": END_OFFSET,
749+
"includeTrailingDelimiter": INCLUDE_TRAILING_DELIMITER,
744750
"versions": VERSIONS,
745751
"projection": PROJECTION,
746752
"fields": FIELDS,
@@ -754,6 +760,9 @@ def test_list_blobs_w_all_arguments_and_user_project(self):
754760
page_token=PAGE_TOKEN,
755761
prefix=PREFIX,
756762
delimiter=DELIMITER,
763+
start_offset=START_OFFSET,
764+
end_offset=END_OFFSET,
765+
include_trailing_delimiter=INCLUDE_TRAILING_DELIMITER,
757766
versions=VERSIONS,
758767
projection=PROJECTION,
759768
fields=FIELDS,

‎tests/unit/test_client.py‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -970,6 +970,9 @@ def test_list_blobs_w_all_arguments_and_user_project(self):
970970
PAGE_TOKEN = "ABCD"
971971
PREFIX = "subfolder"
972972
DELIMITER = "/"
973+
START_OFFSET = "c"
974+
END_OFFSET = "g"
975+
INCLUDE_TRAILING_DELIMITER = True
973976
VERSIONS = True
974977
PROJECTION = "full"
975978
FIELDS = "items/contentLanguage,nextPageToken"
@@ -978,6 +981,9 @@ def test_list_blobs_w_all_arguments_and_user_project(self):
978981
"pageToken": PAGE_TOKEN,
979982
"prefix": PREFIX,
980983
"delimiter": DELIMITER,
984+
"startOffset": START_OFFSET,
985+
"endOffset": END_OFFSET,
986+
"includeTrailingDelimiter": INCLUDE_TRAILING_DELIMITER,
981987
"versions": VERSIONS,
982988
"projection": PROJECTION,
983989
"fields": FIELDS,
@@ -1001,6 +1007,9 @@ def test_list_blobs_w_all_arguments_and_user_project(self):
10011007
page_token=PAGE_TOKEN,
10021008
prefix=PREFIX,
10031009
delimiter=DELIMITER,
1010+
start_offset=START_OFFSET,
1011+
end_offset=END_OFFSET,
1012+
include_trailing_delimiter=INCLUDE_TRAILING_DELIMITER,
10041013
versions=VERSIONS,
10051014
projection=PROJECTION,
10061015
fields=FIELDS,

0 commit comments

Comments
 (0)