From 556bc7448a67aca40618ce9e57c97d092fa0f52e Mon Sep 17 00:00:00 2001 From: Simon <10131203+gaomeng1900@users.noreply.github.com> Date: Fri, 3 Jul 2026 02:29:06 +0800 Subject: [PATCH] chore(llms): add real-api test for all supported models --- packages/llms/src/live-models.test.ts | 224 ++++++++++++++++++++++++++ 1 file changed, 224 insertions(+) create mode 100644 packages/llms/src/live-models.test.ts diff --git a/packages/llms/src/live-models.test.ts b/packages/llms/src/live-models.test.ts new file mode 100644 index 0000000..39216fb --- /dev/null +++ b/packages/llms/src/live-models.test.ts @@ -0,0 +1,224 @@ +/** + * Live compatibility test — hits real provider APIs. + * + * Purpose: verify request formatting and `modelPatch` (see `utils.ts`) work + * for every model below, across the providers that serve them. Not a + * correctness/quality eval — the tool call is trivial and forced via + * `toolChoiceName`, so a failure here means the request/response shape is + * wrong for that model, not that the model is "dumb". + * + * Tests `OpenAIClient` directly (not the `LLM` retry wrapper), so a failure + * always reflects the very first request/response — no retry can mask it. + * + * Each provider's tests skip (not fail) when its `TESTING_*_KEY` env var is + * absent, so this stays CI-safe. To actually run these, put keys in the + * repo-root `.env`: + * + * TESTING_OPENROUTER_KEY=... + * TESTING_DEEPSEEK_KEY=... + * TESTING_ALIYUN_KEY=... + */ +import { config as dotenvConfig } from 'dotenv' +import { dirname, resolve } from 'path' +import { fileURLToPath } from 'url' +import { describe, expect, it } from 'vitest' +import * as z from 'zod/v4' + +import { OpenAIClient } from './OpenAIClient' +import { parseLLMConfig } from './index' +import type { Message, Tool } from './types' + +const __dirname = dirname(fileURLToPath(import.meta.url)) +dotenvConfig({ path: resolve(__dirname, '../../../.env'), quiet: true }) + +const TEST_TIMEOUT = 30_000 + +/** + * Mirrors `packages/website/src/pages/docs/features/models/page.tsx`. + * This package cannot depend on the website, so the list is duplicated here. + * Keep both lists in sync manually when models are added or renamed. + */ +const MODEL_GROUPS: Record = { + Qwen: [ + 'qwen3.7-max', + 'qwen3.7-plus', + 'qwen3.6-max', + 'qwen3.6-plus', + 'qwen3.6-flash', + 'qwen3.5-plus', + 'qwen3.5-flash', + 'qwen3-max', + ], + OpenAI: [ + 'gpt-5.5', + 'gpt-5.4', + 'gpt-5.4-mini', + 'gpt-5.4-nano', + 'gpt-5.2', + 'gpt-5.1', + 'gpt-5', + 'gpt-5-mini', + 'gpt-4.1', + 'gpt-4.1-mini', + ], + DeepSeek: ['deepseek-v4-pro', 'deepseek-v4-flash', 'deepseek-3.2'], + Google: [ + 'gemini-3.5-flash', + 'gemini-3.1-pro', + 'gemini-3.1-flash-lite', + 'gemini-2.5-pro', + 'gemini-2.5-flash', + ], + Anthropic: [ + 'claude-sonnet-5', + 'claude-opus-4-8', + 'claude-opus-4-7', + 'claude-opus-4-6', + 'claude-opus-4-5', + 'claude-sonnet-4-5', + 'claude-haiku-4-5', + ], + MiniMax: ['MiniMax-M3', 'MiniMax-M2.7', 'MiniMax-M2.5'], + xAI: ['grok-4.3', 'grok-build-0.1'], + MoonshotAI: ['kimi-k2.7-code', 'kimi-k2.6', 'kimi-k2.5'], + 'Z.AI': ['glm-5.2', 'glm-5.1', 'glm-5', 'glm-4.7'], +} + +/** + * OpenRouter lists every model as `/`, lowercase. + * See the commented-out entries in the repo-root `.env` for real examples, + * e.g. `x-ai/grok-4.1-fast`, `qwen/qwen3-coder-next`, `deepseek/deepseek-v3.2-exp`. + */ +const OPENROUTER_VENDOR_SLUG: Record = { + Qwen: 'qwen', + OpenAI: 'openai', + DeepSeek: 'deepseek', + Google: 'google', + Anthropic: 'anthropic', + MiniMax: 'minimax', + xAI: 'x-ai', + MoonshotAI: 'moonshotai', + 'Z.AI': 'z-ai', +} + +/** + * Overrides for models whose OpenRouter id doesn't match the + * `/` heuristic — dated snapshots, "-preview" + * suffixes, "v"-prefixed versions, or dots instead of hyphens in the + * version number. Verified against `GET https://openrouter.ai/api/v1/models` + * on 2026-07-03; re-check when models are added to `MODEL_GROUPS`. + */ +const OPENROUTER_ID_OVERRIDES: Record = { + 'qwen3.6-max': 'qwen/qwen3.6-max-preview', + 'qwen3.5-plus': 'qwen/qwen3.5-plus-20260420', + 'qwen3.5-flash': 'qwen/qwen3.5-flash-02-23', + 'deepseek-3.2': 'deepseek/deepseek-v3.2', + 'gemini-3.1-pro': 'google/gemini-3.1-pro-preview', + 'claude-opus-4-8': 'anthropic/claude-opus-4.8', + 'claude-opus-4-7': 'anthropic/claude-opus-4.7', + 'claude-opus-4-6': 'anthropic/claude-opus-4.6', + 'claude-opus-4-5': 'anthropic/claude-opus-4.5', + 'claude-sonnet-4-5': 'anthropic/claude-sonnet-4.5', + 'claude-haiku-4-5': 'anthropic/claude-haiku-4.5', +} + +/** + * Derive a model's OpenRouter id from its brand (a `MODEL_GROUPS` key) and + * native model name. + */ +function toOpenRouterModelId(brand: string, model: string): string { + if (model in OPENROUTER_ID_OVERRIDES) return OPENROUTER_ID_OVERRIDES[model] + const slug = OPENROUTER_VENDOR_SLUG[brand] + if (!slug) throw new Error(`No OpenRouter vendor slug mapped for brand "${brand}"`) + return `${slug}/${model.toLowerCase()}` +} + +const PROVIDERS = { + openrouter: { + baseURL: 'https://openrouter.ai/api/v1', + apiKey: process.env.TESTING_OPENROUTER_KEY, + }, + aliyun: { + baseURL: 'https://dashscope.aliyuncs.com/compatible-mode/v1', + apiKey: process.env.TESTING_ALIYUN_KEY, + }, + deepseek: { + baseURL: 'https://api.deepseek.com', + apiKey: process.env.TESTING_DEEPSEEK_KEY, + }, +} as const + +const ECHO_TOOL: Tool<{ message: string }, string> = { + description: 'Echo back the given message in uppercase.', + inputSchema: z.object({ message: z.string() }), + execute: async ({ message }) => message.toUpperCase(), +} + +const PROMPT: Message[] = [ + { + role: 'user', + content: 'Call the "echo" tool with message set to "ping". You must call the tool.', + }, +] + +async function expectEchoToolCall(baseURL: string, apiKey: string, model: string) { + const client = new OpenAIClient(parseLLMConfig({ baseURL, apiKey, model })) + const result = await client.invoke(PROMPT, { echo: ECHO_TOOL }, new AbortController().signal, { + toolChoiceName: 'echo', + }) + expect(result.toolResult).toBe('PING') +} + +describe.concurrent('OpenRouter — all listed models', () => { + const { baseURL, apiKey } = PROVIDERS.openrouter + + for (const [brand, models] of Object.entries(MODEL_GROUPS)) { + for (const model of models) { + const id = toOpenRouterModelId(brand, model) + it.skipIf(!apiKey)( + id, + async () => { + await expectEchoToolCall(baseURL, apiKey!, id) + }, + TEST_TIMEOUT + ) + } + } +}) + +// Aliyun native ids that don't match the display name in MODEL_GROUPS. +const ALIYUN_ID_OVERRIDES: Record = { + 'qwen3.6-max': 'qwen3.6-max-preview', +} + +describe.concurrent('Aliyun DashScope — Qwen native', () => { + const { baseURL, apiKey } = PROVIDERS.aliyun + + for (const model of MODEL_GROUPS.Qwen) { + const id = ALIYUN_ID_OVERRIDES[model] ?? model + it.skipIf(!apiKey)( + id, + async () => { + await expectEchoToolCall(baseURL, apiKey!, id) + }, + TEST_TIMEOUT + ) + } +}) + +describe.concurrent('DeepSeek — native', () => { + const { baseURL, apiKey } = PROVIDERS.deepseek + // deepseek-3.2 isn't served on DeepSeek's own API (only via OpenRouter + // resellers) — its official API only accepts deepseek-v4-pro/-flash. + const nativeModels = MODEL_GROUPS.DeepSeek.filter((model) => model !== 'deepseek-3.2') + + for (const model of nativeModels) { + it.skipIf(!apiKey)( + model, + async () => { + await expectEchoToolCall(baseURL, apiKey!, model) + }, + TEST_TIMEOUT + ) + } +})