chore(llms): add real-api test for all supported models
This commit is contained in:
Simon
@@ -0,0 +1,224 @@
|
||||
/**
|
||||
* Live compatibility test — hits real provider APIs.
|
||||
*
|
||||
* Purpose: verify request formatting and `modelPatch` (see `utils.ts`) work
|
||||
* for every model below, across the providers that serve them. Not a
|
||||
* correctness/quality eval — the tool call is trivial and forced via
|
||||
* `toolChoiceName`, so a failure here means the request/response shape is
|
||||
* wrong for that model, not that the model is "dumb".
|
||||
*
|
||||
* Tests `OpenAIClient` directly (not the `LLM` retry wrapper), so a failure
|
||||
* always reflects the very first request/response — no retry can mask it.
|
||||
*
|
||||
* Each provider's tests skip (not fail) when its `TESTING_*_KEY` env var is
|
||||
* absent, so this stays CI-safe. To actually run these, put keys in the
|
||||
* repo-root `.env`:
|
||||
*
|
||||
* TESTING_OPENROUTER_KEY=...
|
||||
* TESTING_DEEPSEEK_KEY=...
|
||||
* TESTING_ALIYUN_KEY=...
|
||||
*/
|
||||
import { config as dotenvConfig } from 'dotenv'
|
||||
import { dirname, resolve } from 'path'
|
||||
import { fileURLToPath } from 'url'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import * as z from 'zod/v4'
|
||||
|
||||
import { OpenAIClient } from './OpenAIClient'
|
||||
import { parseLLMConfig } from './index'
|
||||
import type { Message, Tool } from './types'
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url))
|
||||
dotenvConfig({ path: resolve(__dirname, '../../../.env'), quiet: true })
|
||||
|
||||
const TEST_TIMEOUT = 30_000
|
||||
|
||||
/**
|
||||
* Mirrors `packages/website/src/pages/docs/features/models/page.tsx`.
|
||||
* This package cannot depend on the website, so the list is duplicated here.
|
||||
* Keep both lists in sync manually when models are added or renamed.
|
||||
*/
|
||||
const MODEL_GROUPS: Record<string, string[]> = {
|
||||
Qwen: [
|
||||
'qwen3.7-max',
|
||||
'qwen3.7-plus',
|
||||
'qwen3.6-max',
|
||||
'qwen3.6-plus',
|
||||
'qwen3.6-flash',
|
||||
'qwen3.5-plus',
|
||||
'qwen3.5-flash',
|
||||
'qwen3-max',
|
||||
],
|
||||
OpenAI: [
|
||||
'gpt-5.5',
|
||||
'gpt-5.4',
|
||||
'gpt-5.4-mini',
|
||||
'gpt-5.4-nano',
|
||||
'gpt-5.2',
|
||||
'gpt-5.1',
|
||||
'gpt-5',
|
||||
'gpt-5-mini',
|
||||
'gpt-4.1',
|
||||
'gpt-4.1-mini',
|
||||
],
|
||||
DeepSeek: ['deepseek-v4-pro', 'deepseek-v4-flash', 'deepseek-3.2'],
|
||||
Google: [
|
||||
'gemini-3.5-flash',
|
||||
'gemini-3.1-pro',
|
||||
'gemini-3.1-flash-lite',
|
||||
'gemini-2.5-pro',
|
||||
'gemini-2.5-flash',
|
||||
],
|
||||
Anthropic: [
|
||||
'claude-sonnet-5',
|
||||
'claude-opus-4-8',
|
||||
'claude-opus-4-7',
|
||||
'claude-opus-4-6',
|
||||
'claude-opus-4-5',
|
||||
'claude-sonnet-4-5',
|
||||
'claude-haiku-4-5',
|
||||
],
|
||||
MiniMax: ['MiniMax-M3', 'MiniMax-M2.7', 'MiniMax-M2.5'],
|
||||
xAI: ['grok-4.3', 'grok-build-0.1'],
|
||||
MoonshotAI: ['kimi-k2.7-code', 'kimi-k2.6', 'kimi-k2.5'],
|
||||
'Z.AI': ['glm-5.2', 'glm-5.1', 'glm-5', 'glm-4.7'],
|
||||
}
|
||||
|
||||
/**
|
||||
* OpenRouter lists every model as `<vendor-slug>/<model-id>`, lowercase.
|
||||
* See the commented-out entries in the repo-root `.env` for real examples,
|
||||
* e.g. `x-ai/grok-4.1-fast`, `qwen/qwen3-coder-next`, `deepseek/deepseek-v3.2-exp`.
|
||||
*/
|
||||
const OPENROUTER_VENDOR_SLUG: Record<string, string> = {
|
||||
Qwen: 'qwen',
|
||||
OpenAI: 'openai',
|
||||
DeepSeek: 'deepseek',
|
||||
Google: 'google',
|
||||
Anthropic: 'anthropic',
|
||||
MiniMax: 'minimax',
|
||||
xAI: 'x-ai',
|
||||
MoonshotAI: 'moonshotai',
|
||||
'Z.AI': 'z-ai',
|
||||
}
|
||||
|
||||
/**
|
||||
* Overrides for models whose OpenRouter id doesn't match the
|
||||
* `<vendor-slug>/<lowercased-name>` heuristic — dated snapshots, "-preview"
|
||||
* suffixes, "v"-prefixed versions, or dots instead of hyphens in the
|
||||
* version number. Verified against `GET https://openrouter.ai/api/v1/models`
|
||||
* on 2026-07-03; re-check when models are added to `MODEL_GROUPS`.
|
||||
*/
|
||||
const OPENROUTER_ID_OVERRIDES: Record<string, string> = {
|
||||
'qwen3.6-max': 'qwen/qwen3.6-max-preview',
|
||||
'qwen3.5-plus': 'qwen/qwen3.5-plus-20260420',
|
||||
'qwen3.5-flash': 'qwen/qwen3.5-flash-02-23',
|
||||
'deepseek-3.2': 'deepseek/deepseek-v3.2',
|
||||
'gemini-3.1-pro': 'google/gemini-3.1-pro-preview',
|
||||
'claude-opus-4-8': 'anthropic/claude-opus-4.8',
|
||||
'claude-opus-4-7': 'anthropic/claude-opus-4.7',
|
||||
'claude-opus-4-6': 'anthropic/claude-opus-4.6',
|
||||
'claude-opus-4-5': 'anthropic/claude-opus-4.5',
|
||||
'claude-sonnet-4-5': 'anthropic/claude-sonnet-4.5',
|
||||
'claude-haiku-4-5': 'anthropic/claude-haiku-4.5',
|
||||
}
|
||||
|
||||
/**
|
||||
* Derive a model's OpenRouter id from its brand (a `MODEL_GROUPS` key) and
|
||||
* native model name.
|
||||
*/
|
||||
function toOpenRouterModelId(brand: string, model: string): string {
|
||||
if (model in OPENROUTER_ID_OVERRIDES) return OPENROUTER_ID_OVERRIDES[model]
|
||||
const slug = OPENROUTER_VENDOR_SLUG[brand]
|
||||
if (!slug) throw new Error(`No OpenRouter vendor slug mapped for brand "${brand}"`)
|
||||
return `${slug}/${model.toLowerCase()}`
|
||||
}
|
||||
|
||||
const PROVIDERS = {
|
||||
openrouter: {
|
||||
baseURL: 'https://openrouter.ai/api/v1',
|
||||
apiKey: process.env.TESTING_OPENROUTER_KEY,
|
||||
},
|
||||
aliyun: {
|
||||
baseURL: 'https://dashscope.aliyuncs.com/compatible-mode/v1',
|
||||
apiKey: process.env.TESTING_ALIYUN_KEY,
|
||||
},
|
||||
deepseek: {
|
||||
baseURL: 'https://api.deepseek.com',
|
||||
apiKey: process.env.TESTING_DEEPSEEK_KEY,
|
||||
},
|
||||
} as const
|
||||
|
||||
const ECHO_TOOL: Tool<{ message: string }, string> = {
|
||||
description: 'Echo back the given message in uppercase.',
|
||||
inputSchema: z.object({ message: z.string() }),
|
||||
execute: async ({ message }) => message.toUpperCase(),
|
||||
}
|
||||
|
||||
const PROMPT: Message[] = [
|
||||
{
|
||||
role: 'user',
|
||||
content: 'Call the "echo" tool with message set to "ping". You must call the tool.',
|
||||
},
|
||||
]
|
||||
|
||||
async function expectEchoToolCall(baseURL: string, apiKey: string, model: string) {
|
||||
const client = new OpenAIClient(parseLLMConfig({ baseURL, apiKey, model }))
|
||||
const result = await client.invoke(PROMPT, { echo: ECHO_TOOL }, new AbortController().signal, {
|
||||
toolChoiceName: 'echo',
|
||||
})
|
||||
expect(result.toolResult).toBe('PING')
|
||||
}
|
||||
|
||||
describe.concurrent('OpenRouter — all listed models', () => {
|
||||
const { baseURL, apiKey } = PROVIDERS.openrouter
|
||||
|
||||
for (const [brand, models] of Object.entries(MODEL_GROUPS)) {
|
||||
for (const model of models) {
|
||||
const id = toOpenRouterModelId(brand, model)
|
||||
it.skipIf(!apiKey)(
|
||||
id,
|
||||
async () => {
|
||||
await expectEchoToolCall(baseURL, apiKey!, id)
|
||||
},
|
||||
TEST_TIMEOUT
|
||||
)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
// Aliyun native ids that don't match the display name in MODEL_GROUPS.
|
||||
const ALIYUN_ID_OVERRIDES: Record<string, string> = {
|
||||
'qwen3.6-max': 'qwen3.6-max-preview',
|
||||
}
|
||||
|
||||
describe.concurrent('Aliyun DashScope — Qwen native', () => {
|
||||
const { baseURL, apiKey } = PROVIDERS.aliyun
|
||||
|
||||
for (const model of MODEL_GROUPS.Qwen) {
|
||||
const id = ALIYUN_ID_OVERRIDES[model] ?? model
|
||||
it.skipIf(!apiKey)(
|
||||
id,
|
||||
async () => {
|
||||
await expectEchoToolCall(baseURL, apiKey!, id)
|
||||
},
|
||||
TEST_TIMEOUT
|
||||
)
|
||||
}
|
||||
})
|
||||
|
||||
describe.concurrent('DeepSeek — native', () => {
|
||||
const { baseURL, apiKey } = PROVIDERS.deepseek
|
||||
// deepseek-3.2 isn't served on DeepSeek's own API (only via OpenRouter
|
||||
// resellers) — its official API only accepts deepseek-v4-pro/-flash.
|
||||
const nativeModels = MODEL_GROUPS.DeepSeek.filter((model) => model !== 'deepseek-3.2')
|
||||
|
||||
for (const model of nativeModels) {
|
||||
it.skipIf(!apiKey)(
|
||||
model,
|
||||
async () => {
|
||||
await expectEchoToolCall(baseURL, apiKey!, model)
|
||||
},
|
||||
TEST_TIMEOUT
|
||||
)
|
||||
}
|
||||
})
|
||||
Reference in New Issue
Block a user