Auto-recover when the provider rejects a view_image output

When view_image lands an image content block in the agent session and the
next model call fails because the provider rejects the format (SVG on
Anthropic, anything on a text-only model, etc.), the agent used to die
once the general failsafe parked it and there was no way back.

Recovery flow when _run_cycle catches an input-rejection error
(BadRequestError/NotFoundError/422, by status_code) and the latest
session item is an image-bearing function_call_output:

- pop_item() the offending output (single SDK-public primitive)
- add_items() a replacement function_call_output paired by the original
  call_id, with text content telling the model "view_image is
  unsupported on this scan; do not call it again"
- retry the cycle once with empty input_data

Gated by status_code so unrelated failures (timeouts, 5xx, 429, auth,
network blips) leave session content intact — no false-trigger that
would destroy a valid image during a transient hiccup on a
vision-capable model. Hard cap of 3 strips per cycle so a model that
keeps re-calling view_image despite the instruction text still
terminates.

strip_latest_image_from_session lives in core.sessions next to
open_agent_session — both are session helpers operating only through
the SDK's public Session protocol.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
0xallam
2026-05-25 17:23:20 -07:00
parent a451766d97
commit 7e117bc500
2 changed files with 108 additions and 42 deletions
+25 -2
View File
@@ -14,7 +14,7 @@ from agents.exceptions import AgentsException, MaxTurnsExceeded, UserError
from openai import APIError
from strix.core.inputs import child_initial_input
from strix.core.sessions import open_agent_session
from strix.core.sessions import open_agent_session, strip_latest_image_from_session
if TYPE_CHECKING:
@@ -32,6 +32,8 @@ logger = logging.getLogger(__name__)
StreamEventSink = Callable[[str, Any], None]
_INPUT_REJECTION_CODES = frozenset({400, 404, 422})
async def run_agent_loop(
*,
@@ -321,7 +323,7 @@ async def _run_noninteractive_until_lifecycle(
)
async def _run_cycle(
async def _run_cycle( # noqa: PLR0912
agent: Any,
coordinator: AgentCoordinator,
agent_id: str,
@@ -335,6 +337,8 @@ async def _run_cycle(
event_sink: StreamEventSink | None,
hooks: RunHooks[dict[str, Any]] | None,
) -> RunResultBase | None:
image_strips = 0
while True:
try:
await coordinator.mark_running(agent_id)
stream = Runner.run_streamed(
@@ -359,6 +363,25 @@ async def _run_cycle(
finally:
await coordinator.detach_stream(agent_id, stream)
except Exception as exc:
if (
image_strips < 3
and session is not None
and getattr(exc, "status_code", None) in _INPUT_REJECTION_CODES
):
try:
stripped = await strip_latest_image_from_session(session)
except Exception:
logger.exception("image-strip recovery failed for %s", agent_id)
stripped = False
if stripped:
image_strips += 1
logger.info(
"Stripped latest image from %s session after rejection; retrying (%d)",
agent_id,
image_strips,
)
input_data = []
continue
if not interactive:
raise
if isinstance(exc, MaxTurnsExceeded):
+44 -1
View File
@@ -1,10 +1,53 @@
"""SDK session helpers for Strix agents."""
from pathlib import Path
from __future__ import annotations
from typing import TYPE_CHECKING, cast
from agents.memory import SQLiteSession
if TYPE_CHECKING:
from pathlib import Path
from agents.items import TResponseInputItem
from agents.memory import Session
def open_agent_session(agent_id: str, path: Path) -> SQLiteSession:
path.parent.mkdir(parents=True, exist_ok=True)
return SQLiteSession(session_id=agent_id, db_path=path)
_IMAGE_REJECTED_TEXT = (
"[image rejected by the model — view_image is unsupported on this scan; "
"do not call it again, continue without visual inspection]"
)
async def strip_latest_image_from_session(session: Session) -> bool:
items = await session.get_items()
if not items:
return False
latest = items[-1]
if not isinstance(latest, dict) or latest.get("type") != "function_call_output":
return False
output = latest.get("output")
if not isinstance(output, list):
return False
if not any(isinstance(b, dict) and b.get("type") == "input_image" for b in output):
return False
await session.pop_item()
await session.add_items(
cast(
"list[TResponseInputItem]",
[
{
"type": "function_call_output",
"call_id": latest.get("call_id"),
"output": [{"type": "input_text", "text": _IMAGE_REJECTED_TEXT}],
},
],
),
)
return True