Warn for non-frontier LLM selections
This commit is contained in:
@@ -11,12 +11,13 @@ repos:
|
||||
|
||||
# MyPy for static type checking
|
||||
- repo: https://github.com/pre-commit/mirrors-mypy
|
||||
rev: v1.16.0
|
||||
rev: v1.19.1
|
||||
hooks:
|
||||
- id: mypy
|
||||
additional_dependencies: [
|
||||
types-requests,
|
||||
types-python-dateutil,
|
||||
types-Pygments,
|
||||
pydantic,
|
||||
fastapi,
|
||||
pytest,
|
||||
|
||||
@@ -59,6 +59,19 @@ DEFAULT_MODEL_RETRY = ModelRetrySettings(
|
||||
),
|
||||
)
|
||||
|
||||
RECOMMENDED_MODEL_NAMES = (
|
||||
"openai/gpt-5.4",
|
||||
"anthropic/claude-sonnet-4-6",
|
||||
"vertex_ai/gemini-3-pro-preview",
|
||||
)
|
||||
|
||||
FRONTIER_MODEL_PREFIXES = (
|
||||
"gpt-5",
|
||||
"claude-opus-4",
|
||||
"claude-sonnet-4",
|
||||
"gemini-3",
|
||||
)
|
||||
|
||||
|
||||
def configure_sdk_model_defaults(settings: Settings) -> None:
|
||||
"""Apply Strix config to SDK-native defaults."""
|
||||
@@ -154,6 +167,33 @@ def model_supports_reasoning(model_name: str) -> bool:
|
||||
return bool(entry and entry.get("supports_reasoning"))
|
||||
|
||||
|
||||
def is_recommended_or_frontier_model(model_name: str) -> bool:
|
||||
"""Return whether a model is recommended or in a frontier model family."""
|
||||
return any(
|
||||
_is_recommended_or_frontier_candidate(candidate)
|
||||
for candidate in _normalized_model_candidates(model_name)
|
||||
)
|
||||
|
||||
|
||||
def _normalized_model_candidates(model_name: str) -> tuple[str, ...]:
|
||||
name = model_name.strip().lower()
|
||||
if not name:
|
||||
return ()
|
||||
for prefix in ("litellm/", "any-llm/"):
|
||||
if name.startswith(prefix):
|
||||
name = name[len(prefix) :]
|
||||
break
|
||||
if "/" not in name:
|
||||
return (name,)
|
||||
return (name, name.rsplit("/", 1)[1])
|
||||
|
||||
|
||||
def _is_recommended_or_frontier_candidate(model_name: str) -> bool:
|
||||
if model_name in RECOMMENDED_MODEL_NAMES:
|
||||
return True
|
||||
return any(model_name.startswith(prefix) for prefix in FRONTIER_MODEL_PREFIXES)
|
||||
|
||||
|
||||
def is_known_openai_bare_model(model_name: str) -> bool:
|
||||
import litellm
|
||||
|
||||
|
||||
+4
-1
@@ -28,7 +28,10 @@ class ReportUsageHooks(RunHooks[dict[str, Any]]):
|
||||
|
||||
def __init__(self, *, model: str, max_budget_usd: float | None = None) -> None:
|
||||
import math
|
||||
if max_budget_usd is not None and (not math.isfinite(max_budget_usd) or max_budget_usd <= 0):
|
||||
|
||||
if max_budget_usd is not None and (
|
||||
not math.isfinite(max_budget_usd) or max_budget_usd <= 0
|
||||
):
|
||||
raise ValueError("max_budget_usd must be a finite number greater than 0")
|
||||
self._model = model
|
||||
self._max_budget_usd = max_budget_usd
|
||||
|
||||
@@ -23,9 +23,11 @@ from strix.config import (
|
||||
persist_current,
|
||||
)
|
||||
from strix.config.models import (
|
||||
RECOMMENDED_MODEL_NAMES,
|
||||
StrixProvider,
|
||||
configure_sdk_model_defaults,
|
||||
is_known_openai_bare_model,
|
||||
is_recommended_or_frontier_model,
|
||||
)
|
||||
from strix.core.paths import run_dir_for, runtime_state_dir
|
||||
from strix.interface.cli import run_cli
|
||||
@@ -254,6 +256,32 @@ async def warm_up_llm() -> None:
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
if raw_model and not is_recommended_or_frontier_model(raw_model):
|
||||
warn_text = Text()
|
||||
warn_text.append("MODEL QUALITY WARNING", style="bold yellow")
|
||||
warn_text.append("\n\n", style="white")
|
||||
warn_text.append(f"'{raw_model}'", style="bold cyan")
|
||||
warn_text.append(
|
||||
" is not a recommended frontier model for Strix.\nSecurity scans work best with:\n",
|
||||
style="white",
|
||||
)
|
||||
for recommended_model in RECOMMENDED_MODEL_NAMES:
|
||||
warn_text.append(f"• {recommended_model}\n", style="bold cyan")
|
||||
warn_text.append(
|
||||
"\nYou can continue, but weaker models may miss vulnerabilities "
|
||||
"or produce lower-quality findings.",
|
||||
style="white",
|
||||
)
|
||||
console.print(
|
||||
Panel(
|
||||
warn_text,
|
||||
title="[bold white]STRIX",
|
||||
title_align="left",
|
||||
border_style="yellow",
|
||||
padding=(1, 2),
|
||||
),
|
||||
)
|
||||
|
||||
model = StrixProvider().get_model(raw_model)
|
||||
await asyncio.wait_for(
|
||||
model.get_response(
|
||||
@@ -310,6 +338,7 @@ def _positive_budget(value: str) -> float:
|
||||
except ValueError as exc:
|
||||
raise argparse.ArgumentTypeError(f"invalid float value: {value!r}") from exc
|
||||
import math
|
||||
|
||||
if not math.isfinite(budget) or budget <= 0:
|
||||
raise argparse.ArgumentTypeError("must be a finite number greater than 0")
|
||||
return budget
|
||||
|
||||
@@ -83,7 +83,7 @@ class ChatTextArea(TextArea): # type: ignore[misc]
|
||||
|
||||
super()._on_key(event)
|
||||
|
||||
@on(TextArea.Changed) # type: ignore[misc]
|
||||
@on(TextArea.Changed) # type: ignore[untyped-decorator]
|
||||
def _update_height(self, _event: TextArea.Changed | None = None) -> None:
|
||||
if not self.parent:
|
||||
return
|
||||
@@ -1549,7 +1549,7 @@ class StrixTUIApp(App): # type: ignore[misc]
|
||||
|
||||
return AgentMessageRenderer.render_simple(content)
|
||||
|
||||
@on(Tree.NodeHighlighted) # type: ignore[misc]
|
||||
@on(Tree.NodeHighlighted) # type: ignore[untyped-decorator]
|
||||
def handle_tree_highlight(self, event: Tree.NodeHighlighted) -> None:
|
||||
if len(self.screen_stack) > 1 or self.show_splash:
|
||||
return
|
||||
@@ -1569,7 +1569,7 @@ class StrixTUIApp(App): # type: ignore[misc]
|
||||
if agent_id:
|
||||
self.selected_agent_id = agent_id
|
||||
|
||||
@on(Tree.NodeSelected) # type: ignore[misc]
|
||||
@on(Tree.NodeSelected) # type: ignore[untyped-decorator]
|
||||
def handle_tree_node_selected(self, event: Tree.NodeSelected) -> None:
|
||||
if len(self.screen_stack) > 1 or self.show_splash:
|
||||
return
|
||||
|
||||
@@ -48,7 +48,7 @@ logger = logging.getLogger(__name__)
|
||||
class StrixDockerSandboxClient(DockerSandboxClient):
|
||||
# Host directories to bind-mount into the container, set by the docker
|
||||
# backend before ``create()``. Each item is ``{source, target, read_only}``.
|
||||
strix_bind_mounts: list[dict[str, Any]] = [] # overridden per-instance in backends.py
|
||||
strix_bind_mounts: list[dict[str, Any]] | None = None
|
||||
|
||||
async def _create_container(
|
||||
self,
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
"""Tests for LLM model recommendation helpers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from strix.config.models import RECOMMENDED_MODEL_NAMES, is_recommended_or_frontier_model
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_name", RECOMMENDED_MODEL_NAMES)
|
||||
def test_recommended_models_are_accepted(model_name: str) -> None:
|
||||
assert is_recommended_or_frontier_model(model_name)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_name",
|
||||
[
|
||||
"gpt-5.4",
|
||||
"litellm/openai/gpt-5.4",
|
||||
"anthropic/claude-opus-4-1",
|
||||
"any-llm/anthropic/claude-sonnet-4-6",
|
||||
"vertex_ai/gemini-3-pro-preview",
|
||||
],
|
||||
)
|
||||
def test_frontier_model_families_are_accepted(model_name: str) -> None:
|
||||
assert is_recommended_or_frontier_model(model_name)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_name",
|
||||
[
|
||||
"",
|
||||
"openai/gpt-4.1",
|
||||
"anthropic/claude-3-5-sonnet-latest",
|
||||
"ollama/llama3.1",
|
||||
"deepseek/deepseek-chat",
|
||||
],
|
||||
)
|
||||
def test_non_frontier_models_are_rejected(model_name: str) -> None:
|
||||
assert not is_recommended_or_frontier_model(model_name)
|
||||
Reference in New Issue
Block a user