From bb7d1cb68612151ba5c8520df19f5da87b67d4cf Mon Sep 17 00:00:00 2001 From: Rob von Behren Date: Fri, 1 May 2026 12:21:42 -0700 Subject: [PATCH 1/3] Improve devbox polling: eliminate client-side sleep, enable HTTP/2 Replace the client-side busy-poll loop (async_poll_until) for await_running with a new retry_server_poll_until that passes the remaining time budget to the server's wait_for_status endpoint as timeout_seconds. This eliminates a 30-second asyncio.sleep() between each long-poll attempt that was causing devbox startup times to cluster at 40s and 80s instead of the actual 3-5s provisioning time. Enable HTTP/2 by default on AsyncHttpxClient so that concurrent long-poll requests multiplex over fewer TCP connections instead of requiring one connection per in-flight request. For the Trajectory test, this dramatically improved the results with 494 concurrent devboxes: Before: p50=40.5s, p90=48.1s, max=80.9s After: p50=4.4s, p90=9.7s, max=20.6s --- pyproject.toml | 2 +- src/runloop_api_client/_base_client.py | 1 + src/runloop_api_client/lib/polling_async.py | 45 +++++++++++++++++++ .../resources/devboxes/devboxes.py | 24 +++++++--- 4 files changed, 65 insertions(+), 7 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f437210e9..f59d1c37e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ authors = [ ] dependencies = [ - "httpx>=0.23.0, <1", + "httpx[http2]>=0.23.0, <1", "pydantic>=2.0, <3", "typing-extensions>=4.14, <5", "anyio>=3.5.0, <5", diff --git a/src/runloop_api_client/_base_client.py b/src/runloop_api_client/_base_client.py index 7c742dfbc..410e78aab 100644 --- a/src/runloop_api_client/_base_client.py +++ b/src/runloop_api_client/_base_client.py @@ -1375,6 +1375,7 @@ def __init__(self, **kwargs: Any) -> None: kwargs.setdefault("timeout", DEFAULT_TIMEOUT) kwargs.setdefault("limits", DEFAULT_CONNECTION_LIMITS) kwargs.setdefault("follow_redirects", True) + kwargs.setdefault("http2", True) super().__init__(**kwargs) diff --git a/src/runloop_api_client/lib/polling_async.py b/src/runloop_api_client/lib/polling_async.py index 7ba192e86..ff6fe8dea 100644 --- a/src/runloop_api_client/lib/polling_async.py +++ b/src/runloop_api_client/lib/polling_async.py @@ -58,3 +58,48 @@ async def async_poll_until( raise PollingTimeout(f"Exceeded timeout of {config.timeout_seconds} seconds", last_result) await asyncio.sleep(config.interval_seconds) + + +async def retry_server_poll_until( + retriever: Callable[[], Awaitable[T]], + is_terminal: Callable[[T], bool], + timeout_seconds: float = 30.0, + on_error: Optional[Callable[[Exception], T]] = None, +) -> T: + """ + Retry a server-side long-poll until a condition is met or max timeout is reached. + + Args: + retriever: Async or sync callable that returns the object to check. This takes should + take one argument, which is the remaing time to poll.q + is_terminal: Callable that returns True when polling should stop + timeout_seconds: Total time to wait. Must be > 0 + on_error: Optional error handler that can return a value to continue polling + or re-raise the exception to stop polling + + Returns: + The final state of the polled object + + Raises: + PollingTimeout: When max attempts or timeout is reached + """ + start_time = time.time() + last_result: Union[T, None] = None + + while True: + elapsed = time.time() - start_time + remaining_time = timeout_seconds - elapsed + if remaining_time <= 0: + raise PollingTimeout(f"Exceeded timeout of {timeout_seconds} seconds", last_result) + + try: + last_result = await retriever(remaining_time) + except Exception as e: + if on_error is not None: + last_result = on_error(e) + else: + raise + + if is_terminal(last_result): + return last_result + diff --git a/src/runloop_api_client/resources/devboxes/devboxes.py b/src/runloop_api_client/resources/devboxes/devboxes.py index 704d05648..7674c6341 100644 --- a/src/runloop_api_client/resources/devboxes/devboxes.py +++ b/src/runloop_api_client/resources/devboxes/devboxes.py @@ -82,7 +82,7 @@ DiskSnapshotsResourceWithStreamingResponse, AsyncDiskSnapshotsResourceWithStreamingResponse, ) -from ...lib.polling_async import async_poll_until +from ...lib.polling_async import async_poll_until, retry_server_poll_until from ...types.devbox_view import DevboxView from ...types.tunnel_view import TunnelView from ...types.shared_params.mount import Mount @@ -2042,11 +2042,10 @@ async def await_running( Args: id: The ID of the devbox to wait for - config: Optional polling configuration + polling_config: Optional polling configuration extra_headers: Send extra headers extra_query: Add additional query parameters to the request extra_body: Add additional JSON properties to the request - timeout: Override the client-level default timeout for this request, in seconds Returns: The devbox in running state @@ -2056,13 +2055,14 @@ async def await_running( RunloopError: If devbox enters a non-running terminal state """ - async def wait_for_devbox_status() -> DevboxView: + async def wait_for_devbox_status(remaining_timeout_seconds) -> DevboxView: # This wait_for_status endpoint polls the devbox status for 10 seconds until it reaches either running or failure. # If it's neither, it will throw an error. try: return await self._post( f"/v1/devboxes/{id}/wait_for_status", - body={"statuses": ["running", "failure", "shutdown"]}, + body={"statuses": ["running", "failure", "shutdown"], + "timeout_seconds": remaining_timeout_seconds }, cast_to=DevboxView, ) except (APITimeoutError, APIStatusError) as error: @@ -2077,7 +2077,19 @@ async def wait_for_devbox_status() -> DevboxView: def is_done_booting(devbox: DevboxView) -> bool: return devbox.status not in DEVBOX_BOOTING_STATES - devbox = await async_poll_until(wait_for_devbox_status, is_done_booting, polling_config) + # calculate the timeout to use. The PollingConfig doesn't + # match the semantics for server-side polling well, so we + # instead convert interval*attempts to a total time, and take + # the minimum total. + config = polling_config + if not config: + config = PollingConfig() # use defaults + + timeout = config.interval_seconds * config.max_attempts + if config.timeout_seconds is not None and config.timeout_seconds > 0: + timeout = min(config.timeout_seconds, timeout) + + devbox = await retry_server_poll_until(wait_for_devbox_status, is_done_booting, timeout) if devbox.status != "running": raise RunloopError(f"Devbox entered non-running terminal state: {devbox.status}") From dd124dcf3f9bd593f3dd430b4dbd14ff3c78a8fc Mon Sep 17 00:00:00 2001 From: Rob von Behren Date: Fri, 1 May 2026 12:52:52 -0700 Subject: [PATCH 2/3] fmt --- src/runloop_api_client/lib/polling_async.py | 11 +++-- .../resources/devboxes/devboxes.py | 2 +- uv.lock | 42 +++++++++++++++++-- 3 files changed, 45 insertions(+), 10 deletions(-) diff --git a/src/runloop_api_client/lib/polling_async.py b/src/runloop_api_client/lib/polling_async.py index ff6fe8dea..0a55dab82 100644 --- a/src/runloop_api_client/lib/polling_async.py +++ b/src/runloop_api_client/lib/polling_async.py @@ -61,7 +61,7 @@ async def async_poll_until( async def retry_server_poll_until( - retriever: Callable[[], Awaitable[T]], + retriever: Callable[[float], Awaitable[T]], is_terminal: Callable[[T], bool], timeout_seconds: float = 30.0, on_error: Optional[Callable[[Exception], T]] = None, @@ -70,8 +70,8 @@ async def retry_server_poll_until( Retry a server-side long-poll until a condition is met or max timeout is reached. Args: - retriever: Async or sync callable that returns the object to check. This takes should - take one argument, which is the remaing time to poll.q + retriever: Async callable that takes the remaining timeout (seconds) and + returns the object to check. is_terminal: Callable that returns True when polling should stop timeout_seconds: Total time to wait. Must be > 0 on_error: Optional error handler that can return a value to continue polling @@ -83,12 +83,11 @@ async def retry_server_poll_until( Raises: PollingTimeout: When max attempts or timeout is reached """ - start_time = time.time() last_result: Union[T, None] = None + start_time = time.time() while True: - elapsed = time.time() - start_time - remaining_time = timeout_seconds - elapsed + remaining_time = timeout_seconds - (time.time() - start_time) if remaining_time <= 0: raise PollingTimeout(f"Exceeded timeout of {timeout_seconds} seconds", last_result) diff --git a/src/runloop_api_client/resources/devboxes/devboxes.py b/src/runloop_api_client/resources/devboxes/devboxes.py index 7674c6341..be4fc99e4 100644 --- a/src/runloop_api_client/resources/devboxes/devboxes.py +++ b/src/runloop_api_client/resources/devboxes/devboxes.py @@ -2055,7 +2055,7 @@ async def await_running( RunloopError: If devbox enters a non-running terminal state """ - async def wait_for_devbox_status(remaining_timeout_seconds) -> DevboxView: + async def wait_for_devbox_status(remaining_timeout_seconds: float) -> DevboxView: # This wait_for_status endpoint polls the devbox status for 10 seconds until it reaches either running or failure. # If it's neither, it will throw an error. try: diff --git a/uv.lock b/uv.lock index b35c4f478..88dc754a1 100644 --- a/uv.lock +++ b/uv.lock @@ -1043,6 +1043,28 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, ] +[[package]] +name = "h2" +version = "4.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "hpack" }, + { name = "hyperframe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1d/17/afa56379f94ad0fe8defd37d6eb3f89a25404ffc71d4d848893d270325fc/h2-4.3.0.tar.gz", hash = "sha256:6c59efe4323fa18b47a632221a1888bd7fde6249819beda254aeca909f221bf1", size = 2152026, upload-time = "2025-08-23T18:12:19.778Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/69/b2/119f6e6dcbd96f9069ce9a2665e0146588dc9f88f29549711853645e736a/h2-4.3.0-py3-none-any.whl", hash = "sha256:c438f029a25f7945c69e0ccf0fb951dc3f73a5f6412981daee861431b70e2bdd", size = 61779, upload-time = "2025-08-23T18:12:17.779Z" }, +] + +[[package]] +name = "hpack" +version = "4.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2c/48/71de9ed269fdae9c8057e5a4c0aa7402e8bb16f2c6e90b3aa53327b113f8/hpack-4.1.0.tar.gz", hash = "sha256:ec5eca154f7056aa06f196a557655c5b009b382873ac8d1e66e79e87535f1dca", size = 51276, upload-time = "2025-01-22T21:44:58.347Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/07/c6/80c95b1b2b94682a72cbdbfb85b81ae2daffa4291fbfa1b1464502ede10d/hpack-4.1.0-py3-none-any.whl", hash = "sha256:157ac792668d995c657d93111f46b4535ed114f0c9c8d672271bbec7eae1b496", size = 34357, upload-time = "2025-01-22T21:44:56.92Z" }, +] + [[package]] name = "html5lib" version = "1.1" @@ -1084,6 +1106,11 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, ] +[package.optional-dependencies] +http2 = [ + { name = "h2" }, +] + [[package]] name = "httpx-aiohttp" version = "0.1.12" @@ -1097,6 +1124,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/16/8d/85c9701e9af72ca132a1783e2a54364a90c6da832304416a30fc11196ab2/httpx_aiohttp-0.1.12-py3-none-any.whl", hash = "sha256:5b0eac39a7f360fa7867a60bcb46bb1024eada9c01cbfecdb54dc1edb3fb7141", size = 6367, upload-time = "2025-12-12T10:12:14.018Z" }, ] +[[package]] +name = "hyperframe" +version = "6.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/02/e7/94f8232d4a74cc99514c13a9f995811485a6903d48e5d952771ef6322e30/hyperframe-6.1.0.tar.gz", hash = "sha256:f630908a00854a7adeabd6382b43923a4c4cd4b821fcb527e6ab9e15382a3b08", size = 26566, upload-time = "2025-01-22T21:41:49.302Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/48/30/47d0bf6072f7252e6521f3447ccfa40b421b6824517f82854703d0f5a98b/hyperframe-6.1.0-py3-none-any.whl", hash = "sha256:b03380493a519fce58ea5af42e4a42317bf9bd425596f7a0835ffce80f1a42e5", size = 13007, upload-time = "2025-01-22T21:41:47.295Z" }, +] + [[package]] name = "idna" version = "3.11" @@ -2386,12 +2422,12 @@ wheels = [ [[package]] name = "runloop-api-client" -version = "1.19.0" +version = "1.20.0" source = { editable = "." } dependencies = [ { name = "anyio" }, { name = "distro" }, - { name = "httpx" }, + { name = "httpx", extra = ["http2"] }, { name = "pydantic" }, { name = "sniffio" }, { name = "typing-extensions" }, @@ -2444,7 +2480,7 @@ requires-dist = [ { name = "aiohttp", marker = "extra == 'aiohttp'" }, { name = "anyio", specifier = ">=3.5.0,<5" }, { name = "distro", specifier = ">=1.7.0,<2" }, - { name = "httpx", specifier = ">=0.23.0,<1" }, + { name = "httpx", extras = ["http2"], specifier = ">=0.23.0,<1" }, { name = "httpx-aiohttp", marker = "extra == 'aiohttp'", specifier = ">=0.1.9" }, { name = "pydantic", specifier = ">=2.0,<3" }, { name = "sniffio" }, From f7933c7e1e8c7fefd706b1bc67d80826774f1a0b Mon Sep 17 00:00:00 2001 From: Rob von Behren Date: Fri, 1 May 2026 13:08:46 -0700 Subject: [PATCH 3/3] fmt --- src/runloop_api_client/lib/polling_async.py | 1 - src/runloop_api_client/resources/devboxes/devboxes.py | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/runloop_api_client/lib/polling_async.py b/src/runloop_api_client/lib/polling_async.py index 0a55dab82..9bc1bb752 100644 --- a/src/runloop_api_client/lib/polling_async.py +++ b/src/runloop_api_client/lib/polling_async.py @@ -101,4 +101,3 @@ async def retry_server_poll_until( if is_terminal(last_result): return last_result - diff --git a/src/runloop_api_client/resources/devboxes/devboxes.py b/src/runloop_api_client/resources/devboxes/devboxes.py index be4fc99e4..96628e2bb 100644 --- a/src/runloop_api_client/resources/devboxes/devboxes.py +++ b/src/runloop_api_client/resources/devboxes/devboxes.py @@ -2061,8 +2061,7 @@ async def wait_for_devbox_status(remaining_timeout_seconds: float) -> DevboxView try: return await self._post( f"/v1/devboxes/{id}/wait_for_status", - body={"statuses": ["running", "failure", "shutdown"], - "timeout_seconds": remaining_timeout_seconds }, + body={"statuses": ["running", "failure", "shutdown"], "timeout_seconds": remaining_timeout_seconds}, cast_to=DevboxView, ) except (APITimeoutError, APIStatusError) as error: