Expand frontier LLM model recommendations

This commit is contained in:
Devin AI
2026-06-26 16:32:54 +00:00
parent 8ca83e4c16
commit 4c9f56fcd5
2 changed files with 92 additions and 23 deletions
+71 -19
View File
@@ -60,26 +60,39 @@ DEFAULT_MODEL_RETRY = ModelRetrySettings(
)
RECOMMENDED_MODEL_NAMES = (
"openai/gpt-5.5",
"openai/gpt-5.5-pro",
"openai/gpt-5.4",
"openai/gpt-5.4-pro",
"openai/gpt-5.3-codex",
"anthropic/claude-opus-4-8",
"anthropic/claude-sonnet-4-6",
"vertex_ai/gemini-3-pro-preview",
"vertex_ai/gemini-3.1-pro-preview",
"gemini/gemini-3.1-pro-preview",
"xai/grok-4.3",
"deepseek/deepseek-v4-pro",
"deepseek/deepseek-reasoner",
"dashscope/qwen3-max-2026-01-23",
"moonshot/kimi-k2.7-code",
"moonshot/kimi-k2.6",
"mistral/mistral-medium-3-5",
"mistral/magistral-medium-latest",
)
_RECOMMENDED_MODEL_NAME_SET = frozenset(name.lower() for name in RECOMMENDED_MODEL_NAMES)
FRONTIER_MODEL_PROVIDERS = (
"openai",
"anthropic",
"vertex_ai",
"gemini",
"google",
)
FRONTIER_MODEL_PREFIXES = (
"gpt-5",
"claude-opus-4",
"claude-sonnet-4",
"gemini-3",
FRONTIER_MODEL_FAMILIES = (
(("azure", "azure_ai", "bedrock_mantle", "openai"), ("gpt-5",)),
(
("anthropic", "azure_ai", "bedrock", "claude", "databricks", "snowflake", "vertex_ai"),
("claude-opus-4", "claude-sonnet-4"),
),
(("google", "gemini", "vertex_ai"), ("gemini-3",)),
(("xai", "x-ai"), ("grok-4",)),
(("deepseek",), ("deepseek-v4", "deepseek-r1", "deepseek-reasoner")),
(("alibaba", "dashscope", "qwen"), ("qwen3.7", "qwen3.5", "qwen3-max")),
(("moonshot", "moonshotai", "kimi"), ("kimi-k2.7", "kimi-k2.6", "kimi-k2.5")),
(("mistral", "mistralai"), ("mistral-medium-3-5", "magistral-medium")),
)
@@ -185,9 +198,10 @@ def is_recommended_or_frontier_model(model_name: str) -> bool:
if name in _RECOMMENDED_MODEL_NAME_SET:
return True
provider_name, bare_model_name = _split_model_provider(name)
if provider_name is None:
return _is_frontier_model_name(bare_model_name)
return provider_name in FRONTIER_MODEL_PROVIDERS and _is_frontier_model_name(bare_model_name)
return any(
_matches_frontier_family(provider_name, bare_model_name, provider_markers, prefixes)
for provider_markers, prefixes in FRONTIER_MODEL_FAMILIES
)
def _normalized_model_name(model_name: str) -> str:
@@ -206,8 +220,46 @@ def _split_model_provider(model_name: str) -> tuple[str | None, str]:
return provider_name, bare_model_name
def _is_frontier_model_name(model_name: str) -> bool:
return any(model_name.startswith(prefix) for prefix in FRONTIER_MODEL_PREFIXES)
def _matches_frontier_family(
provider_name: str | None,
model_name: str,
provider_markers: tuple[str, ...],
model_prefixes: tuple[str, ...],
) -> bool:
if not _matches_model_prefix(model_name, model_prefixes):
return False
if provider_name is None:
return True
return _contains_provider_marker(
provider_name, provider_markers, split_compound_names=True
) or _contains_provider_marker(model_name, provider_markers)
def _matches_model_prefix(model_name: str, model_prefixes: tuple[str, ...]) -> bool:
return any(
candidate.startswith(prefix)
for candidate in _model_name_candidates(model_name)
for prefix in model_prefixes
)
def _model_name_candidates(model_name: str) -> tuple[str, ...]:
if "." not in model_name:
return (model_name,)
suffixes = tuple(
model_name.split(".", index)[-1] for index in range(1, model_name.count(".") + 1)
)
return (model_name, *suffixes)
def _contains_provider_marker(
value: str, provider_markers: tuple[str, ...], *, split_compound_names: bool = False
) -> bool:
parts = set(value.replace(".", "/").split("/"))
if split_compound_names:
for separator in ("_", "-"):
parts.update(piece for part in tuple(parts) for piece in part.split(separator))
return any(marker in parts for marker in provider_markers)
def is_known_openai_bare_model(model_name: str) -> bool:
+21 -4
View File
@@ -19,11 +19,27 @@ def test_recommended_models_are_matched_case_insensitively() -> None:
@pytest.mark.parametrize(
"model_name",
[
"gpt-5.4",
"litellm/openai/gpt-5.4",
"anthropic/claude-opus-4-1",
"gpt-5.5",
"litellm/openai/gpt-5.4-pro",
"azure_ai/gpt-5.5-pro",
"bedrock_mantle/openai.gpt-5.5",
"anthropic/claude-opus-4-8",
"anthropic.claude-opus-4-8",
"vertex_ai/claude-sonnet-4-6@default",
"any-llm/anthropic/claude-sonnet-4-6",
"vertex_ai/gemini-3-pro-preview",
"vertex_ai/gemini-3.1-pro-preview",
"openrouter/google/gemini-3.1-pro-preview",
"xai/grok-4.3",
"openrouter/x-ai/grok-4",
"deepseek/deepseek-v4-pro",
"deepseek/deepseek-r1-0528",
"deepseek/deepseek-reasoner",
"dashscope/qwen3-max-2026-01-23",
"qwen3.7-max",
"moonshot/kimi-k2.6",
"kimi-k2.7-code",
"mistral/mistral-medium-3-5",
"mistral/magistral-medium-latest",
],
)
def test_frontier_model_families_are_accepted(model_name: str) -> None:
@@ -39,6 +55,7 @@ def test_frontier_model_families_are_accepted(model_name: str) -> None:
"ollama/llama3.1",
"deepseek/deepseek-chat",
"custom-ollama/gpt-5-mini-local",
"custom-provider/claude-opus-4-local",
],
)
def test_non_frontier_models_are_rejected(model_name: str) -> None: