Skip to content

Commit ac20367

Browse files
vdusekclaude
vdusek
and
claude
committed
refactor: Separate initial and max timeouts with lower defaults
Introduce `timeout_max` parameter to cap exponential timeout growth independently from the initial timeout. Lower defaults to `DEFAULT_REQUEST_TIMEOUT=10s`, `DEFAULT_REQUEST_TIMEOUT_MAX=600s`, and `DEFAULT_MAX_RETRIES=4`, producing the sequence [10, 20, 40, 80, 160]. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent f1c77b1 commit ac20367

5 files changed

Lines changed: 54 additions & 28 deletions

File tree

‎src/apify_client/_apify_client.py‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
DEFAULT_MAX_RETRIES,
1111
DEFAULT_MIN_DELAY_BETWEEN_RETRIES,
1212
DEFAULT_REQUEST_TIMEOUT,
13+
DEFAULT_REQUEST_TIMEOUT_MAX,
1314
)
1415
from apify_client._docs import docs_group
1516
from apify_client._http_clients import HttpClient, HttpClientAsync, ImpitHttpClient, ImpitHttpClientAsync
@@ -115,6 +116,7 @@ def __init__(
115116
max_retries: int = DEFAULT_MAX_RETRIES,
116117
min_delay_between_retries: timedelta = DEFAULT_MIN_DELAY_BETWEEN_RETRIES,
117118
timeout: timedelta = DEFAULT_REQUEST_TIMEOUT,
119+
timeout_max: timedelta = DEFAULT_REQUEST_TIMEOUT_MAX,
118120
headers: dict[str, str] | None = None,
119121
) -> None:
120122
"""Initialize the Apify API client.
@@ -132,7 +134,8 @@ def __init__(
132134
max_retries: How many times to retry a failed request at most.
133135
min_delay_between_retries: How long will the client wait between retrying requests
134136
(increases exponentially from this value).
135-
timeout: The socket timeout of the HTTP requests sent to the Apify API.
137+
timeout: The initial socket timeout of the HTTP requests sent to the Apify API.
138+
timeout_max: Maximum timeout cap for exponential timeout growth across retries.
136139
headers: Additional HTTP headers to include in all API requests.
137140
"""
138141
# We need to do this because of mocking in tests and default mutable arguments.
@@ -192,6 +195,7 @@ def __init__(
192195
self._max_retries = max_retries
193196
self._min_delay_between_retries = min_delay_between_retries
194197
self._timeout = timeout
198+
self._timeout_max = timeout_max
195199
self._headers = headers
196200

197201
@classmethod
@@ -250,6 +254,7 @@ def http_client(self) -> HttpClient:
250254
self._http_client = ImpitHttpClient(
251255
token=self._token,
252256
timeout=self._timeout,
257+
timeout_max=self._timeout_max,
253258
max_retries=self._max_retries,
254259
min_delay_between_retries=self._min_delay_between_retries,
255260
statistics=self._statistics,
@@ -456,6 +461,7 @@ def __init__(
456461
max_retries: int = DEFAULT_MAX_RETRIES,
457462
min_delay_between_retries: timedelta = DEFAULT_MIN_DELAY_BETWEEN_RETRIES,
458463
timeout: timedelta = DEFAULT_REQUEST_TIMEOUT,
464+
timeout_max: timedelta = DEFAULT_REQUEST_TIMEOUT_MAX,
459465
headers: dict[str, str] | None = None,
460466
) -> None:
461467
"""Initialize the Apify API client.
@@ -473,7 +479,8 @@ def __init__(
473479
max_retries: How many times to retry a failed request at most.
474480
min_delay_between_retries: How long will the client wait between retrying requests
475481
(increases exponentially from this value).
476-
timeout: The socket timeout of the HTTP requests sent to the Apify API.
482+
timeout: The initial socket timeout of the HTTP requests sent to the Apify API.
483+
timeout_max: Maximum timeout cap for exponential timeout growth across retries.
477484
headers: Additional HTTP headers to include in all API requests.
478485
"""
479486
# We need to do this because of mocking in tests and default mutable arguments.
@@ -533,6 +540,7 @@ def __init__(
533540
self._max_retries = max_retries
534541
self._min_delay_between_retries = min_delay_between_retries
535542
self._timeout = timeout
543+
self._timeout_max = timeout_max
536544
self._headers = headers
537545

538546
@classmethod
@@ -591,6 +599,7 @@ def http_client(self) -> HttpClientAsync:
591599
self._http_client = ImpitHttpClientAsync(
592600
token=self._token,
593601
timeout=self._timeout,
602+
timeout_max=self._timeout_max,
594603
max_retries=self._max_retries,
595604
min_delay_between_retries=self._min_delay_between_retries,
596605
statistics=self._statistics,

‎src/apify_client/_consts.py‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,10 +23,13 @@
2323
API_VERSION = 'v2'
2424
"""Current Apify API version."""
2525

26-
DEFAULT_REQUEST_TIMEOUT = timedelta(seconds=30)
27-
"""Default timeout for individual API requests."""
26+
DEFAULT_REQUEST_TIMEOUT = timedelta(seconds=10)
27+
"""Default initial timeout for individual API requests."""
2828

29-
DEFAULT_MAX_RETRIES = 8
29+
DEFAULT_REQUEST_TIMEOUT_MAX = timedelta(seconds=600)
30+
"""Default maximum timeout cap for individual API requests (limits exponential growth)."""
31+
32+
DEFAULT_MAX_RETRIES = 4
3033
"""Default maximum number of retries for failed requests."""
3134

3235
DEFAULT_MIN_DELAY_BETWEEN_RETRIES = timedelta(milliseconds=500)

‎src/apify_client/_http_clients/_base.py‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
DEFAULT_MAX_RETRIES,
1515
DEFAULT_MIN_DELAY_BETWEEN_RETRIES,
1616
DEFAULT_REQUEST_TIMEOUT,
17+
DEFAULT_REQUEST_TIMEOUT_MAX,
1718
Timeout,
1819
)
1920
from apify_client._docs import docs_group
@@ -90,6 +91,7 @@ def __init__(
9091
*,
9192
token: str | None = None,
9293
timeout: timedelta = DEFAULT_REQUEST_TIMEOUT,
94+
timeout_max: timedelta = DEFAULT_REQUEST_TIMEOUT_MAX,
9395
max_retries: int = DEFAULT_MAX_RETRIES,
9496
min_delay_between_retries: timedelta = DEFAULT_MIN_DELAY_BETWEEN_RETRIES,
9597
statistics: ClientStatistics | None = None,
@@ -99,13 +101,15 @@ def __init__(
99101
100102
Args:
101103
token: Apify API token for authentication.
102-
timeout: Request timeout.
104+
timeout: Initial request timeout.
105+
timeout_max: Maximum timeout cap for exponential timeout growth across retries.
103106
max_retries: Maximum number of retries for failed requests.
104107
min_delay_between_retries: Minimum delay between retries.
105108
statistics: Statistics tracker for API calls. Created automatically if not provided.
106109
headers: Additional HTTP headers to include in all requests.
107110
"""
108111
self._timeout = timeout
112+
self._timeout_max = timeout_max
109113
self._max_retries = max_retries
110114
self._min_delay_between_retries = min_delay_between_retries
111115
self._statistics = statistics or ClientStatistics()

‎src/apify_client/_http_clients/_impit.py‎

Lines changed: 18 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
DEFAULT_MAX_RETRIES,
1515
DEFAULT_MIN_DELAY_BETWEEN_RETRIES,
1616
DEFAULT_REQUEST_TIMEOUT,
17+
DEFAULT_REQUEST_TIMEOUT_MAX,
1718
Timeout,
1819
)
1920
from apify_client._docs import docs_group
@@ -54,18 +55,19 @@ def _compute_timeout(
5455
client_timeout: timedelta,
5556
attempt: int,
5657
timeout: Timeout,
58+
timeout_max: timedelta,
5759
) -> float:
58-
"""Compute timeout for a request attempt with exponential increase, bounded by client timeout.
60+
"""Compute timeout for a request attempt with exponential increase, bounded by timeout_max.
5961
6062
For `'no_timeout'`, returns 0 (impit interprets this as no timeout). For `None`, uses the client-level timeout.
61-
For `timedelta` values, doubles the timeout with each attempt but caps at the client-level timeout.
63+
For `timedelta` values, doubles the timeout with each attempt but caps at the maximum timeout.
6264
"""
6365
if timeout == 'no_timeout':
6466
return 0
6567

6668
timeout_secs = to_seconds(timeout or client_timeout)
67-
client_timeout_secs = to_seconds(client_timeout)
68-
return min(client_timeout_secs, timeout_secs * 2 ** (attempt - 1))
69+
timeout_max_secs = to_seconds(timeout_max)
70+
return min(timeout_max_secs, timeout_secs * 2 ** (attempt - 1))
6971

7072

7173
@docs_group('HTTP clients')
@@ -82,6 +84,7 @@ def __init__(
8284
*,
8385
token: str | None = None,
8486
timeout: timedelta = DEFAULT_REQUEST_TIMEOUT,
87+
timeout_max: timedelta = DEFAULT_REQUEST_TIMEOUT_MAX,
8588
max_retries: int = DEFAULT_MAX_RETRIES,
8689
min_delay_between_retries: timedelta = DEFAULT_MIN_DELAY_BETWEEN_RETRIES,
8790
statistics: ClientStatistics | None = None,
@@ -91,7 +94,8 @@ def __init__(
9194
9295
Args:
9396
token: Apify API token for authentication.
94-
timeout: Default timeout for HTTP requests.
97+
timeout: Default initial timeout for HTTP requests.
98+
timeout_max: Maximum timeout cap for exponential timeout growth across retries.
9599
max_retries: Maximum number of retry attempts for failed requests.
96100
min_delay_between_retries: Minimum delay between retries (increases exponentially with each attempt).
97101
statistics: Statistics tracker for API calls. Created automatically if not provided.
@@ -100,6 +104,7 @@ def __init__(
100104
super().__init__(
101105
token=token,
102106
timeout=timeout,
107+
timeout_max=timeout_max,
103108
max_retries=max_retries,
104109
min_delay_between_retries=min_delay_between_retries,
105110
statistics=statistics,
@@ -109,7 +114,7 @@ def __init__(
109114
self._impit_client = impit.Client(
110115
headers=self._headers,
111116
follow_redirects=True,
112-
timeout=to_seconds(self._timeout),
117+
timeout=to_seconds(self._timeout_max),
113118
)
114119

115120
def call(
@@ -212,7 +217,7 @@ def _make_request(
212217
url=url_with_params,
213218
headers=headers,
214219
content=content,
215-
timeout=_compute_timeout(self._timeout, attempt, timeout),
220+
timeout=_compute_timeout(self._timeout, attempt, timeout, self._timeout_max),
216221
stream=stream or False,
217222
)
218223

@@ -309,6 +314,7 @@ def __init__(
309314
*,
310315
token: str | None = None,
311316
timeout: timedelta = DEFAULT_REQUEST_TIMEOUT,
317+
timeout_max: timedelta = DEFAULT_REQUEST_TIMEOUT_MAX,
312318
max_retries: int = DEFAULT_MAX_RETRIES,
313319
min_delay_between_retries: timedelta = DEFAULT_MIN_DELAY_BETWEEN_RETRIES,
314320
statistics: ClientStatistics | None = None,
@@ -318,7 +324,8 @@ def __init__(
318324
319325
Args:
320326
token: Apify API token for authentication.
321-
timeout: Default timeout for HTTP requests.
327+
timeout: Default initial timeout for HTTP requests.
328+
timeout_max: Maximum timeout cap for exponential timeout growth across retries.
322329
max_retries: Maximum number of retry attempts for failed requests.
323330
min_delay_between_retries: Minimum delay between retries (increases exponentially with each attempt).
324331
statistics: Statistics tracker for API calls. Created automatically if not provided.
@@ -327,6 +334,7 @@ def __init__(
327334
super().__init__(
328335
token=token,
329336
timeout=timeout,
337+
timeout_max=timeout_max,
330338
max_retries=max_retries,
331339
min_delay_between_retries=min_delay_between_retries,
332340
statistics=statistics,
@@ -336,7 +344,7 @@ def __init__(
336344
self._impit_async_client = impit.AsyncClient(
337345
headers=self._headers,
338346
follow_redirects=True,
339-
timeout=to_seconds(self._timeout),
347+
timeout=to_seconds(self._timeout_max),
340348
)
341349

342350
async def call(
@@ -439,7 +447,7 @@ async def _make_request(
439447
url=url_with_params,
440448
headers=headers,
441449
content=content,
442-
timeout=_compute_timeout(self._timeout, attempt, timeout),
450+
timeout=_compute_timeout(self._timeout, attempt, timeout, self._timeout_max),
443451
stream=stream or False,
444452
)
445453

‎tests/unit/test_client_timeouts.py‎

Lines changed: 14 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -77,12 +77,12 @@ async def test_none_timeout_uses_client_default_async(patch_request: list) -> No
7777
async def test_dynamic_timeout_async_client(monkeypatch: pytest.MonkeyPatch) -> None:
7878
"""Tests timeout values for request with retriable errors.
7979
80-
Values should increase with each attempt, starting from initial call value and bounded by the client timeout value.
80+
Values should increase with each attempt, starting from initial call value and bounded by timeout_max.
8181
"""
8282
should_raise_error = iter((True, True, True, False))
8383
call_timeout = 1
84-
client_timeout = 5
85-
expected_timeouts = [call_timeout, 2, 4, client_timeout]
84+
timeout_max = 5
85+
expected_timeouts = [call_timeout, 2, 4, timeout_max]
8686
retry_counter_mock = Mock()
8787

8888
timeouts = []
@@ -98,9 +98,10 @@ async def mock_request(*_args: Any, **kwargs: Any) -> Response:
9898

9999
monkeypatch.setattr('impit.AsyncClient.request', mock_request)
100100

101-
response = await ImpitHttpClientAsync(timeout=timedelta(seconds=client_timeout)).call(
102-
method='GET', url='http://placeholder.url/async_timeout', timeout=timedelta(seconds=call_timeout)
103-
)
101+
response = await ImpitHttpClientAsync(
102+
timeout=timedelta(seconds=call_timeout),
103+
timeout_max=timedelta(seconds=timeout_max),
104+
).call(method='GET', url='http://placeholder.url/async_timeout', timeout=timedelta(seconds=call_timeout))
104105

105106
# Check that the retry counter was called the expected number of times
106107
# (4 times: 3 retries + 1 final successful call)
@@ -141,12 +142,12 @@ async def mock_request(*_args: Any, **_kwargs: Any) -> Response:
141142
def test_dynamic_timeout_sync_client(monkeypatch: pytest.MonkeyPatch) -> None:
142143
"""Tests timeout values for request with retriable errors.
143144
144-
Values should increase with each attempt, starting from initial call value and bounded by the client timeout value.
145+
Values should increase with each attempt, starting from initial call value and bounded by timeout_max.
145146
"""
146147
should_raise_error = iter((True, True, True, False))
147148
call_timeout = 1
148-
client_timeout = 5
149-
expected_timeouts = [call_timeout, 2, 4, client_timeout]
149+
timeout_max = 5
150+
expected_timeouts = [call_timeout, 2, 4, timeout_max]
150151
retry_counter_mock = Mock()
151152

152153
timeouts = []
@@ -162,9 +163,10 @@ def mock_request(*_args: Any, **kwargs: Any) -> Response:
162163

163164
monkeypatch.setattr('impit.Client.request', mock_request)
164165

165-
response = ImpitHttpClient(timeout=timedelta(seconds=client_timeout)).call(
166-
method='GET', url='http://placeholder.url/sync_timeout', timeout=timedelta(seconds=call_timeout)
167-
)
166+
response = ImpitHttpClient(
167+
timeout=timedelta(seconds=call_timeout),
168+
timeout_max=timedelta(seconds=timeout_max),
169+
).call(method='GET', url='http://placeholder.url/sync_timeout', timeout=timedelta(seconds=call_timeout))
168170

169171
# Check that the retry counter was called the expected number of times
170172
# (4 times: 3 retries + 1 final successful call)

0 commit comments

Comments
 (0)