Skip to content

Commit 07a676e

Browse files
authored
fix: add retry coverage to the streaming portion of a download (#241)
* apply retry strategy to Download.consume block * change to implement retry across entire operation by adding wrapper around Download.consume * apply _consume_with_retries across Download sub classes * add unit test * add tests * update docstrings
1 parent cf51d3f commit 07a676e

File tree

4 files changed

+355
-23
lines changed

4 files changed

+355
-23
lines changed

packages/google-resumable-media/google/resumable_media/_download.py

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
"""Virtual bases classes for downloading media from Google APIs."""
1616

1717

18+
import functools
1819
import re
1920

2021
from six.moves import http_client
@@ -208,6 +209,37 @@ def consume(self, transport, timeout=None):
208209
"""
209210
raise NotImplementedError(u"This implementation is virtual.")
210211

212+
def _consume_with_retries(self, func, transport, timeout=None):
213+
"""Attempts to retry a download and consume until success.
214+
215+
The consume ``func`` is retry-able based on the HTTP response and
216+
connection error type. Retry covers both the initial response and
217+
the streaming portion of the download.
218+
219+
Will retry until :meth:`~.RetryStrategy.retry_allowed` (on the current
220+
``retry_strategy``) returns :data:`False`.
221+
222+
Args:
223+
func (Callable): A callable that takes no arguments and produces
224+
an HTTP response which will be checked as retry-able.
225+
transport (object): An object which can make authenticated
226+
requests.
227+
timeout (Optional[Union[float, Tuple[float, float]]]):
228+
The number of seconds to wait for the server response.
229+
Depending on the retry strategy, a request may be repeated
230+
several times using the same timeout each time.
231+
232+
Can also be passed as a tuple (connect_timeout, read_timeout).
233+
See :meth:`requests.Session.request` documentation for details.
234+
235+
Returns:
236+
~requests.Response: The return value of ``transport.request()``.
237+
"""
238+
func_to_retry = functools.partial(func, transport=transport, timeout=timeout)
239+
return _helpers.wait_and_retry(
240+
func_to_retry, self._get_status_code, self._retry_strategy
241+
)
242+
211243

212244
class ChunkedDownload(DownloadBase):
213245
"""Download a resource in chunks from a Google API.
@@ -439,6 +471,37 @@ def consume_next_chunk(self, transport, timeout=None):
439471
"""
440472
raise NotImplementedError(u"This implementation is virtual.")
441473

474+
def _consume_with_retries(self, func, transport, timeout=None):
475+
"""Attempts to retry a consume_next_chunk until success.
476+
477+
The consume_next_chunk ``func`` is retry-able based on the HTTP response
478+
and connection error type. Retry covers both the initial response and
479+
the streaming portion of the download.
480+
481+
Will retry until :meth:`~.RetryStrategy.retry_allowed` (on the current
482+
``retry_strategy``) returns :data:`False`.
483+
484+
Args:
485+
func (Callable): A callable that takes no arguments and produces
486+
an HTTP response which will be checked as retry-able.
487+
transport (object): An object which can make authenticated
488+
requests.
489+
timeout (Optional[Union[float, Tuple[float, float]]]):
490+
The number of seconds to wait for the server response.
491+
Depending on the retry strategy, a request may be repeated
492+
several times using the same timeout each time.
493+
494+
Can also be passed as a tuple (connect_timeout, read_timeout).
495+
See :meth:`requests.Session.request` documentation for details.
496+
497+
Returns:
498+
~requests.Response: The return value of ``transport.request()``.
499+
"""
500+
func_to_retry = functools.partial(func, transport=transport, timeout=timeout)
501+
return _helpers.wait_and_retry(
502+
func_to_retry, self._get_status_code, self._retry_strategy
503+
)
504+
442505

443506
def add_bytes_range(start, end, headers):
444507
"""Add a bytes range to a header dictionary.

0 commit comments

Comments
 (0)