From 94c361cbb6b3ee627e2c701ebaa2f4f586bc2cb9 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 26 Jun 2026 13:10:50 +0000 Subject: [PATCH] Warn for non-frontier LLM selections --- .pre-commit-config.yaml | 3 ++- strix/config/models.py | 40 ++++++++++++++++++++++++++++++++++ strix/core/hooks.py | 5 ++++- strix/interface/main.py | 29 ++++++++++++++++++++++++ strix/interface/tui/app.py | 6 ++--- strix/runtime/docker_client.py | 2 +- tests/test_models.py | 40 ++++++++++++++++++++++++++++++++++ 7 files changed, 119 insertions(+), 6 deletions(-) create mode 100644 tests/test_models.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ab2b556..02d9120 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,12 +11,13 @@ repos: # MyPy for static type checking - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.16.0 + rev: v1.19.1 hooks: - id: mypy additional_dependencies: [ types-requests, types-python-dateutil, + types-Pygments, pydantic, fastapi, pytest, diff --git a/strix/config/models.py b/strix/config/models.py index 7f6227f..3f8f497 100644 --- a/strix/config/models.py +++ b/strix/config/models.py @@ -59,6 +59,19 @@ DEFAULT_MODEL_RETRY = ModelRetrySettings( ), ) +RECOMMENDED_MODEL_NAMES = ( + "openai/gpt-5.4", + "anthropic/claude-sonnet-4-6", + "vertex_ai/gemini-3-pro-preview", +) + +FRONTIER_MODEL_PREFIXES = ( + "gpt-5", + "claude-opus-4", + "claude-sonnet-4", + "gemini-3", +) + def configure_sdk_model_defaults(settings: Settings) -> None: """Apply Strix config to SDK-native defaults.""" @@ -154,6 +167,33 @@ def model_supports_reasoning(model_name: str) -> bool: return bool(entry and entry.get("supports_reasoning")) +def is_recommended_or_frontier_model(model_name: str) -> bool: + """Return whether a model is recommended or in a frontier model family.""" + return any( + _is_recommended_or_frontier_candidate(candidate) + for candidate in _normalized_model_candidates(model_name) + ) + + +def _normalized_model_candidates(model_name: str) -> tuple[str, ...]: + name = model_name.strip().lower() + if not name: + return () + for prefix in ("litellm/", "any-llm/"): + if name.startswith(prefix): + name = name[len(prefix) :] + break + if "/" not in name: + return (name,) + return (name, name.rsplit("/", 1)[1]) + + +def _is_recommended_or_frontier_candidate(model_name: str) -> bool: + if model_name in RECOMMENDED_MODEL_NAMES: + return True + return any(model_name.startswith(prefix) for prefix in FRONTIER_MODEL_PREFIXES) + + def is_known_openai_bare_model(model_name: str) -> bool: import litellm diff --git a/strix/core/hooks.py b/strix/core/hooks.py index 6b0d592..64562d7 100644 --- a/strix/core/hooks.py +++ b/strix/core/hooks.py @@ -28,7 +28,10 @@ class ReportUsageHooks(RunHooks[dict[str, Any]]): def __init__(self, *, model: str, max_budget_usd: float | None = None) -> None: import math - if max_budget_usd is not None and (not math.isfinite(max_budget_usd) or max_budget_usd <= 0): + + if max_budget_usd is not None and ( + not math.isfinite(max_budget_usd) or max_budget_usd <= 0 + ): raise ValueError("max_budget_usd must be a finite number greater than 0") self._model = model self._max_budget_usd = max_budget_usd diff --git a/strix/interface/main.py b/strix/interface/main.py index 4eae052..2ed6e50 100644 --- a/strix/interface/main.py +++ b/strix/interface/main.py @@ -23,9 +23,11 @@ from strix.config import ( persist_current, ) from strix.config.models import ( + RECOMMENDED_MODEL_NAMES, StrixProvider, configure_sdk_model_defaults, is_known_openai_bare_model, + is_recommended_or_frontier_model, ) from strix.core.paths import run_dir_for, runtime_state_dir from strix.interface.cli import run_cli @@ -254,6 +256,32 @@ async def warm_up_llm() -> None: ) sys.exit(1) + if raw_model and not is_recommended_or_frontier_model(raw_model): + warn_text = Text() + warn_text.append("MODEL QUALITY WARNING", style="bold yellow") + warn_text.append("\n\n", style="white") + warn_text.append(f"'{raw_model}'", style="bold cyan") + warn_text.append( + " is not a recommended frontier model for Strix.\nSecurity scans work best with:\n", + style="white", + ) + for recommended_model in RECOMMENDED_MODEL_NAMES: + warn_text.append(f"• {recommended_model}\n", style="bold cyan") + warn_text.append( + "\nYou can continue, but weaker models may miss vulnerabilities " + "or produce lower-quality findings.", + style="white", + ) + console.print( + Panel( + warn_text, + title="[bold white]STRIX", + title_align="left", + border_style="yellow", + padding=(1, 2), + ), + ) + model = StrixProvider().get_model(raw_model) await asyncio.wait_for( model.get_response( @@ -310,6 +338,7 @@ def _positive_budget(value: str) -> float: except ValueError as exc: raise argparse.ArgumentTypeError(f"invalid float value: {value!r}") from exc import math + if not math.isfinite(budget) or budget <= 0: raise argparse.ArgumentTypeError("must be a finite number greater than 0") return budget diff --git a/strix/interface/tui/app.py b/strix/interface/tui/app.py index 4e43d97..05d920d 100644 --- a/strix/interface/tui/app.py +++ b/strix/interface/tui/app.py @@ -83,7 +83,7 @@ class ChatTextArea(TextArea): # type: ignore[misc] super()._on_key(event) - @on(TextArea.Changed) # type: ignore[misc] + @on(TextArea.Changed) # type: ignore[untyped-decorator] def _update_height(self, _event: TextArea.Changed | None = None) -> None: if not self.parent: return @@ -1549,7 +1549,7 @@ class StrixTUIApp(App): # type: ignore[misc] return AgentMessageRenderer.render_simple(content) - @on(Tree.NodeHighlighted) # type: ignore[misc] + @on(Tree.NodeHighlighted) # type: ignore[untyped-decorator] def handle_tree_highlight(self, event: Tree.NodeHighlighted) -> None: if len(self.screen_stack) > 1 or self.show_splash: return @@ -1569,7 +1569,7 @@ class StrixTUIApp(App): # type: ignore[misc] if agent_id: self.selected_agent_id = agent_id - @on(Tree.NodeSelected) # type: ignore[misc] + @on(Tree.NodeSelected) # type: ignore[untyped-decorator] def handle_tree_node_selected(self, event: Tree.NodeSelected) -> None: if len(self.screen_stack) > 1 or self.show_splash: return diff --git a/strix/runtime/docker_client.py b/strix/runtime/docker_client.py index 497ae2f..d41a5a3 100644 --- a/strix/runtime/docker_client.py +++ b/strix/runtime/docker_client.py @@ -48,7 +48,7 @@ logger = logging.getLogger(__name__) class StrixDockerSandboxClient(DockerSandboxClient): # Host directories to bind-mount into the container, set by the docker # backend before ``create()``. Each item is ``{source, target, read_only}``. - strix_bind_mounts: list[dict[str, Any]] = [] # overridden per-instance in backends.py + strix_bind_mounts: list[dict[str, Any]] | None = None async def _create_container( self, diff --git a/tests/test_models.py b/tests/test_models.py new file mode 100644 index 0000000..f5ec974 --- /dev/null +++ b/tests/test_models.py @@ -0,0 +1,40 @@ +"""Tests for LLM model recommendation helpers.""" + +from __future__ import annotations + +import pytest + +from strix.config.models import RECOMMENDED_MODEL_NAMES, is_recommended_or_frontier_model + + +@pytest.mark.parametrize("model_name", RECOMMENDED_MODEL_NAMES) +def test_recommended_models_are_accepted(model_name: str) -> None: + assert is_recommended_or_frontier_model(model_name) + + +@pytest.mark.parametrize( + "model_name", + [ + "gpt-5.4", + "litellm/openai/gpt-5.4", + "anthropic/claude-opus-4-1", + "any-llm/anthropic/claude-sonnet-4-6", + "vertex_ai/gemini-3-pro-preview", + ], +) +def test_frontier_model_families_are_accepted(model_name: str) -> None: + assert is_recommended_or_frontier_model(model_name) + + +@pytest.mark.parametrize( + "model_name", + [ + "", + "openai/gpt-4.1", + "anthropic/claude-3-5-sonnet-latest", + "ollama/llama3.1", + "deepseek/deepseek-chat", + ], +) +def test_non_frontier_models_are_rejected(model_name: str) -> None: + assert not is_recommended_or_frontier_model(model_name)