From 60abc09ff95c831f503a0bbf73cdabf6c0fdd20c Mon Sep 17 00:00:00 2001 From: seanturner83 Date: Wed, 22 Apr 2026 21:26:47 +0100 Subject: [PATCH] fix: wrap acompletion in asyncio.wait_for to prevent indefinite hangs (#453) * fix: wrap acompletion in asyncio.wait_for to prevent indefinite hangs litellm's timeout parameter doesn't always propagate to the underlying httpx transport for Bedrock converse streaming. When Bedrock accepts the TCP connection but never starts streaming chunks, the acompletion call hangs indefinitely with all connections in CLOSED state. This wraps the acompletion call in asyncio.wait_for() using the configured LLM_TIMEOUT (default 300s). TimeoutError is already retryable via _should_retry (status_code=None), so the retry loop handles it. Diagnosed via faulthandler thread dump showing the main asyncio event loop blocked in selectors.select() with no pending callbacks. Co-Authored-By: Claude Opus 4.6 * fix: add per-chunk timeout to streaming loop Addresses review feedback: the initial asyncio.wait_for only guards the acompletion call. If Bedrock returns headers but stalls mid-stream, the async for loop could still hang indefinitely. Replaces async for with explicit __anext__ calls wrapped in asyncio.wait_for, using the same configured timeout. Mid-stream stalls now raise TimeoutError and trigger the existing retry logic. Co-Authored-By: Claude Opus 4.6 --------- Co-authored-by: Sean Turner Co-authored-by: Claude Opus 4.6 --- strix/llm/llm.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/strix/llm/llm.py b/strix/llm/llm.py index 4f62495..5e6a01f 100644 --- a/strix/llm/llm.py +++ b/strix/llm/llm.py @@ -176,9 +176,18 @@ class LLM: done_streaming = 0 self._total_stats.requests += 1 - response = await acompletion(**self._build_completion_args(messages), stream=True) + timeout = self.config.timeout + response = await asyncio.wait_for( + acompletion(**self._build_completion_args(messages), stream=True), + timeout=timeout, + ) - async for chunk in response: + async_iter = response.__aiter__() + while True: + try: + chunk = await asyncio.wait_for(async_iter.__anext__(), timeout=timeout) + except StopAsyncIteration: + break chunks.append(chunk) if done_streaming: done_streaming += 1