From e9c1af03f62c21f17a2c5b59aeaee51f39a2a3c9 Mon Sep 17 00:00:00 2001 From: jackson-jia-914 Date: Wed, 24 Jun 2026 18:02:25 +0800 Subject: [PATCH] feat(llm): add disableThinking option to suppress reasoning in extraction models (#228) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Support multiple inference engines and model providers via a strategy enum so each receives its own thinking-disabling field in chat-completion request bodies. Strategies: - "vllm" → chat_template_kwargs: { enable_thinking: false } (vLLM / SGLang) - "deepseek" → enable_thinking: false (DeepSeek official API) - "dashscope" → enable_thinking: false (Alibaba DashScope / Qwen) - "openai" → reasoning_effort: "low" (OpenAI o-series, cannot fully disable) - "anthropic" → thinking: { type: "disabled" } (Anthropic Claude) - "kimi" → thinking: { type: "disabled" } (Kimi / Moonshot) - "gemini" → thinking_config: { thinking_budget: 0 } (Google Gemini) Defaults to false (no wrapper). Also accepts boolean true as a shorthand for "vllm" for convenience. Covered paths: - StandaloneLLMRunner (OpenClaw plugin + gateway): llm.disableThinking, wired through parseConfig, tdai-core, seed-runtime, and gateway config (TDAI_LLM_DISABLE_THINKING env also accepts strategy names). - Offload local mode (L1/L1.5/L2): separate offload.disableThinking. The fetch wrapper lives in src/utils/no-think-fetch.ts with a STRATEGY_TRANSFORMERS map for clean dispatch. StandaloneLLMRunner builds it once in the constructor; LocalLlmClient caches it at construction time and passes it through to callLlm(). Add vitest unit tests for all 7 strategies, normalization, validation, embedding skip, and non-JSON tolerance (20 tests total). Signed-off-by: jackson.jia --- CHANGELOG.md | 5 + openclaw.plugin.json | 4 +- src/adapters/standalone/llm-runner.ts | 16 ++ src/config.ts | 30 +++ src/core/seed/seed-runtime.ts | 1 + src/core/tdai-core.ts | 1 + src/gateway/config.ts | 26 +++ src/gateway/server.ts | 1 + src/offload/index.ts | 2 +- src/offload/local-llm/index.ts | 13 ++ src/offload/local-llm/llm-caller.ts | 13 ++ src/utils/no-think-fetch.test.ts | 269 ++++++++++++++++++++++++++ src/utils/no-think-fetch.ts | 135 +++++++++++++ 13 files changed, 514 insertions(+), 2 deletions(-) create mode 100644 src/utils/no-think-fetch.test.ts create mode 100644 src/utils/no-think-fetch.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ad6ed0..1df12b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,11 @@ - **L0 JSONL 分片日**和 **cleaner 清理边界**跟随配置时区（默认仍为系统时区）。 - 存储层（SQLite / TCVDB）时间戳始终为 UTC instant，**无需数据迁移**。 - 统一收敛原有 4 处分散的时间格式化 helper 到 `src/utils/time.ts`，减少代码重复。 +- **关闭推理模型 thinking（`disableThinking`）**：新增 `llm.disableThinking` 与 `offload.disableThinking` 两项配置（均默认 `false`，不改变现有行为），支持多种推理引擎/模型提供商的关闭方式。 + - 可选策略：`"vllm"` (vLLM/SGLang, `chat_template_kwargs.enable_thinking=false`)、`"deepseek"` (DeepSeek API, 顶层 `enable_thinking=false`)、`"dashscope"` (阿里云 DashScope/Qwen, 顶层 `enable_thinking=false`)、`"openai"` (OpenAI o系列, `reasoning_effort="low"`)、`"anthropic"` (Anthropic Claude, `thinking.type="disabled"`)、`"kimi"` (Kimi/Moonshot, `thinking.type="disabled"`)、`"gemini"` (Google Gemini, `thinking_config.thinking_budget=0`)。 + - 环境变量 `TDAI_LLM_DISABLE_THINKING` 支持策略名（如 `deepseek`）和布尔值。 + - 修复 offload local-llm 模式下每次 LLM 调用都重新创建 fetch wrapper 的性能问题（现在在 `LocalLlmClient` 构造函数中创建一次并缓存）。 + - 注入逻辑抽取到 `src/utils/no-think-fetch.ts` 共享，新增 vitest 单测覆盖全部策略 / 跳过 embedding / 非 JSON 容错。 ### ⚠️ 升级注意（仅在显式配置 `timezone` 时生效） diff --git a/openclaw.plugin.json b/openclaw.plugin.json index 6d79877..0a6d7e9 100644 --- a/openclaw.plugin.json +++ b/openclaw.plugin.json @@ -140,7 +140,8 @@ "apiKey": { "type": "string", "description": "API Key" }, "model": { "type": "string", "default": "gpt-4o", "description": "模型名称（如 gpt-4o, deepseek-v3, claude-sonnet-4-6）" }, "maxTokens": { "type": "number", "default": 4096, "description": "最大输出 token 数" }, - "timeoutMs": { "type": "number", "default": 120000, "description": "请求超时（毫秒）" } + "timeoutMs": { "type": "number", "default": 120000, "description": "请求超时（毫秒）" }, + "disableThinking": { "oneOf": [{ "type": "boolean" }, { "type": "string", "enum": ["vllm", "deepseek", "dashscope", "openai", "anthropic", "kimi", "gemini"] }], "default": false, "description": "禁用推理模型思考过程。可选策略: \"vllm\" (vLLM/SGLang), \"deepseek\" (DeepSeek API), \"dashscope\" (阿里云 DashScope/Qwen), \"openai\" (OpenAI o系列), \"anthropic\" (Anthropic Claude), \"kimi\" (Kimi/Moonshot), \"gemini\" (Google Gemini)。" } } }, "offload": { @@ -150,6 +151,7 @@ "enabled": { "type": "boolean", "default": false, "description": "是否启用 Context Offload（默认关闭，不影响 Memory 功能）" }, "model": { "type": "string", "description": "Offload 使用的 LLM 模型（格式: provider/model），未填写时使用 openclaw 默认模型" }, "temperature": { "type": "number", "default": 0.2, "description": "LLM 温度参数" }, + "disableThinking": { "oneOf": [{ "type": "boolean" }, { "type": "string", "enum": ["vllm", "deepseek", "dashscope", "openai", "anthropic", "kimi", "gemini"] }], "default": false, "description": "禁用推理模型思考过程（仅 local 模式）。可选策略: \"vllm\"/\"deepseek\"/\"dashscope\"/\"openai\"/\"anthropic\"/\"kimi\"/\"gemini\"。" }, "forceTriggerThreshold": { "type": "number", "default": 4, "description": "累积多少个 tool pair 后强制触发 L1" }, "dataDir": { "type": "string", "description": "自定义数据目录（绝对路径），默认 ~/.openclaw/context-offload" }, "defaultContextWindow": { "type": "number", "default": 200000, "description": "默认上下文窗口大小" }, diff --git a/src/adapters/standalone/llm-runner.ts b/src/adapters/standalone/llm-runner.ts index 13bf620..cf127ca 100644 --- a/src/adapters/standalone/llm-runner.ts +++ b/src/adapters/standalone/llm-runner.ts @@ -21,6 +21,7 @@ import path from "node:path"; import { generateText, tool, stepCountIs, jsonSchema } from "ai"; import { createOpenAI } from "@ai-sdk/openai"; import { report } from "../../core/report/reporter.js"; +import { createNoThinkFetch, type DisableThinkingStrategy } from "../../utils/no-think-fetch.js"; import type { LLMRunner, LLMRunParams, @@ -49,6 +50,16 @@ export interface StandaloneLLMConfig { maxTokens?: number; /** Request timeout in milliseconds (default: 120_000). */ timeoutMs?: number; + /** + * Controls how thinking/reasoning is disabled for the LLM endpoint. + * - `false` (default): no thinking-disabling wrapper + * - `"vllm"`: vLLM/SGLang chat_template_kwargs + * - `"deepseek"` / `"dashscope"`: top-level enable_thinking: false + * - `"openai"`: reasoning_effort: "low" + * - `"anthropic"` / `"kimi"`: thinking: { type: "disabled" } + * - `"gemini"`: thinking_config: { thinking_budget: 0 } + */ + disableThinking?: DisableThinkingStrategy; } // ============================ @@ -158,6 +169,7 @@ export class StandaloneLLMRunner implements LLMRunner { private model: string; private enableTools: boolean; private logger?: Logger; + private readonly customFetch?: typeof globalThis.fetch; constructor(opts: { config: StandaloneLLMConfig; @@ -169,6 +181,9 @@ export class StandaloneLLMRunner implements LLMRunner { this.model = opts.model ?? opts.config.model; this.enableTools = opts.enableTools ?? false; this.logger = opts.logger; + this.customFetch = opts.config.disableThinking + ? createNoThinkFetch(opts.config.disableThinking) + : undefined; } async run(params: LLMRunParams): Promise { @@ -189,6 +204,7 @@ export class StandaloneLLMRunner implements LLMRunner { baseURL: this.config.baseUrl, apiKey: this.config.apiKey, compatibility: "compatible", + ...(this.customFetch ? { fetch: this.customFetch } : {}), }); // For pure text tasks like L1 extraction, avoid exposing any tools. diff --git a/src/config.ts b/src/config.ts index ea60c53..ef61441 100644 --- a/src/config.ts +++ b/src/config.ts @@ -7,6 +7,9 @@ * Minimal config (zero config): {} — all fields have sensible defaults. */ +import type { DisableThinkingStrategy } from "./utils/no-think-fetch.js"; +import { normalizeDisableThinking } from "./utils/no-think-fetch.js"; + // ============================ // Type definitions // ============================ @@ -203,6 +206,17 @@ export interface StandaloneLLMOverrideConfig { maxTokens: number; /** Request timeout in milliseconds (default: 120000). */ timeoutMs: number; + /** + * Controls how thinking/reasoning is disabled for the LLM endpoint (default: false). + * - `false`: no thinking-disabling wrapper (default) + * - `"vllm"`: vLLM/SGLang — `chat_template_kwargs: { enable_thinking: false }` + * - `"deepseek"`: DeepSeek official API — top-level `enable_thinking: false` + * - `"dashscope"`: Alibaba DashScope (Qwen) — top-level `enable_thinking: false` + * - `"openai"`: OpenAI o-series — `reasoning_effort: "low"` (cannot fully disable) + * - `"anthropic"` / `"kimi"`: Anthropic Claude / Kimi (Moonshot) — `thinking: { type: "disabled" }` + * - `"gemini"`: Google Gemini — `thinking_config: { thinking_budget: 0 }` + */ + disableThinking: DisableThinkingStrategy; } /** Context Offload settings — controls multi-layer context compression. */ @@ -222,6 +236,12 @@ export interface OffloadConfig { model?: string; /** LLM temperature (default: 0.2) */ temperature: number; + /** + * Controls how thinking/reasoning is disabled for the offload local-mode LLM (default: false). + * See `StandaloneLLMOverrideConfig.disableThinking` for the full list of strategies. + * Applies only to `mode: "local"`. + */ + disableThinking: DisableThinkingStrategy; /** Force-trigger L1 when pending tool pairs >= this threshold (default: 4) */ forceTriggerThreshold: number; /** Custom data directory (absolute path). Default: ~/.openclaw/context-offload */ @@ -459,6 +479,7 @@ export function parseConfig(raw: Record | undefined): MemoryTda mode: offloadMode, model: optStr(offloadGroup, "model"), temperature: num(offloadGroup, "temperature") ?? 0.2, + disableThinking: normalizeDisableThinking(boolOrStr(offloadGroup, "disableThinking")), forceTriggerThreshold: num(offloadGroup, "forceTriggerThreshold") ?? 4, dataDir: optStr(offloadGroup, "dataDir"), defaultContextWindow: num(offloadGroup, "defaultContextWindow") ?? 200000, @@ -561,6 +582,7 @@ export function parseConfig(raw: Record | undefined): MemoryTda model: str(llmGroup, "model") ?? "gpt-4o", maxTokens: num(llmGroup, "maxTokens") ?? 4096, timeoutMs: num(llmGroup, "timeoutMs") ?? 120_000, + disableThinking: normalizeDisableThinking(boolOrStr(llmGroup, "disableThinking")), }; })(), offload, @@ -597,6 +619,14 @@ function bool(src: Record, key: string): boolean | undefined { return typeof v === "boolean" ? v : undefined; } +/** Read a field that may be boolean or string. */ +function boolOrStr(src: Record, key: string): boolean | string | undefined { + const v = src[key]; + if (typeof v === "boolean") return v; + if (typeof v === "string" && v.trim()) return v.trim(); + return undefined; +} + function strArray(src: Record, key: string): string[] | undefined { const v = src[key]; if (!Array.isArray(v)) return undefined; diff --git a/src/core/seed/seed-runtime.ts b/src/core/seed/seed-runtime.ts index 46e6647..3cfeb89 100644 --- a/src/core/seed/seed-runtime.ts +++ b/src/core/seed/seed-runtime.ts @@ -87,6 +87,7 @@ async function createSeedPipeline(opts: SeedRuntimeOptions): Promise<{ pipeline: model: cfg.llm.model, maxTokens: cfg.llm.maxTokens, timeoutMs: cfg.llm.timeoutMs, + disableThinking: cfg.llm.disableThinking, }, logger, }); diff --git a/src/core/tdai-core.ts b/src/core/tdai-core.ts index 977d4a2..9185e51 100644 --- a/src/core/tdai-core.ts +++ b/src/core/tdai-core.ts @@ -440,6 +440,7 @@ export class TdaiCore { model: this.cfg.llm.model, maxTokens: this.cfg.llm.maxTokens, timeoutMs: this.cfg.llm.timeoutMs, + disableThinking: this.cfg.llm.disableThinking, }, logger: this.logger, }); diff --git a/src/gateway/config.ts b/src/gateway/config.ts index c6161fe..4fff581 100644 --- a/src/gateway/config.ts +++ b/src/gateway/config.ts @@ -14,6 +14,7 @@ import YAML from "yaml"; import { getEnv } from "../utils/env.js"; import { parseConfig as parseMemoryConfig } from "../config.js"; import type { MemoryTdaiConfig } from "../config.js"; +import { normalizeDisableThinking } from "../utils/no-think-fetch.js"; import type { StandaloneLLMConfig } from "../adapters/standalone/llm-runner.js"; // ============================ @@ -132,6 +133,9 @@ export function loadGatewayConfig(overrides?: Partial): GatewayCo model: env("TDAI_LLM_MODEL") ?? str(llmConfig, "model") ?? "gpt-4o", maxTokens: envInt("TDAI_LLM_MAX_TOKENS") ?? num(llmConfig, "maxTokens") ?? 4096, timeoutMs: envInt("TDAI_LLM_TIMEOUT_MS") ?? num(llmConfig, "timeoutMs") ?? 120_000, + disableThinking: normalizeDisableThinking( + envBoolOrStr("TDAI_LLM_DISABLE_THINKING") ?? boolOrStr(llmConfig, "disableThinking") + ), }; // Memory config (reuse the plugin's parseConfig for full compatibility) @@ -233,6 +237,28 @@ function envInt(key: string): number | undefined { return Number.isFinite(n) ? n : undefined; } +/** + * Read an env var that may be a boolean ("true"/"false"/"1"/"0") + * or a plain string (strategy name like "deepseek", "anthropic"). + * Returns the lowercase string for strategy names. + */ +function envBoolOrStr(key: string): boolean | string | undefined { + const raw = env(key); + if (raw === undefined) return undefined; + const v = raw.toLowerCase(); + if (v === "true" || v === "1") return true; + if (v === "false" || v === "0") return false; + return v; // lowercase strategy name +} + +/** Read a field that may be boolean or string from a config object. */ +function boolOrStr(src: Record, key: string): boolean | string | undefined { + const v = src[key]; + if (typeof v === "boolean") return v; + if (typeof v === "string" && v.trim()) return v.trim(); + return undefined; +} + function obj(c: Record, key: string): Record { const v = c[key]; return v && typeof v === "object" && !Array.isArray(v) ? v as Record : {}; diff --git a/src/gateway/server.ts b/src/gateway/server.ts index b4c66ec..1da5592 100644 --- a/src/gateway/server.ts +++ b/src/gateway/server.ts @@ -530,6 +530,7 @@ export class TdaiGateway { model: this.config.llm.model, maxTokens: this.config.llm.maxTokens, timeoutMs: this.config.llm.timeoutMs, + disableThinking: this.config.llm.disableThinking, }, }; if (body.config_override) { diff --git a/src/offload/index.ts b/src/offload/index.ts index 7d0d724..35c18bb 100644 --- a/src/offload/index.ts +++ b/src/offload/index.ts @@ -376,7 +376,7 @@ export function registerOffload(api: any, offloadConfig: OffloadConfig): void { if (baseUrl && apiKey) { backendClient = new LocalLlmClient( - { baseUrl, apiKey, model: modelId, temperature: offloadConfig.temperature, timeoutMs: offloadConfig.backendTimeoutMs }, + { baseUrl, apiKey, model: modelId, temperature: offloadConfig.temperature, timeoutMs: offloadConfig.backendTimeoutMs, disableThinking: offloadConfig.disableThinking }, logger, ); } else { diff --git a/src/offload/local-llm/index.ts b/src/offload/local-llm/index.ts index 78b4782..bdd2b3e 100644 --- a/src/offload/local-llm/index.ts +++ b/src/offload/local-llm/index.ts @@ -7,6 +7,7 @@ * Used when `offload.model` is configured and `offload.backendUrl` is not set. */ import { callLlm, type LlmCallerConfig } from "./llm-caller.js"; +import { createNoThinkFetch, type DisableThinkingStrategy } from "../../utils/no-think-fetch.js"; import { L1_SYSTEM_PROMPT, buildL1UserPrompt, type L1ToolPair } from "./prompts/l1-prompt.js"; import { L15_SYSTEM_PROMPT, buildL15UserPrompt, type L15CurrentMmd, type L15MmdMeta } from "./prompts/l15-prompt.js"; import { L2_SYSTEM_PROMPT, buildL2UserPrompt, type L2NewEntry } from "./prompts/l2-prompt.js"; @@ -24,11 +25,13 @@ export interface LocalLlmClientConfig { model: string; temperature?: number; timeoutMs?: number; + disableThinking?: DisableThinkingStrategy; } export class LocalLlmClient { private config: LlmCallerConfig; private logger?: PluginLogger; + private readonly customFetch?: typeof globalThis.fetch; constructor(cfg: LocalLlmClientConfig, logger?: PluginLogger) { this.config = { @@ -37,8 +40,15 @@ export class LocalLlmClient { model: cfg.model, temperature: cfg.temperature ?? 0.2, timeoutMs: cfg.timeoutMs ?? 120_000, + disableThinking: cfg.disableThinking ?? false, }; this.logger = logger; + + // Cache the fetch wrapper at construction time — avoids per-call creation. + this.customFetch = cfg.disableThinking + ? createNoThinkFetch(cfg.disableThinking) + : undefined; + logger?.info?.(`${TAG} Initialized: model=${cfg.model}, baseUrl=${cfg.baseUrl}`); } @@ -59,6 +69,7 @@ export class LocalLlmClient { systemPrompt: L1_SYSTEM_PROMPT, userPrompt, label: "L1", + customFetch: this.customFetch, }, this.logger); const entries = parseL1Response(raw); @@ -97,6 +108,7 @@ export class LocalLlmClient { systemPrompt: L15_SYSTEM_PROMPT, userPrompt, label: "L1.5", + customFetch: this.customFetch, }, this.logger); const result = parseL15Response(raw); @@ -138,6 +150,7 @@ export class LocalLlmClient { userPrompt, label: "L2", timeoutMs: 120_000, // L2 may take longer due to complex prompts + customFetch: this.customFetch, }, this.logger); const result = parseL2Response(raw); diff --git a/src/offload/local-llm/llm-caller.ts b/src/offload/local-llm/llm-caller.ts index 95ae691..9f61baa 100644 --- a/src/offload/local-llm/llm-caller.ts +++ b/src/offload/local-llm/llm-caller.ts @@ -6,6 +6,7 @@ */ import { generateText } from "ai"; import { createOpenAI } from "@ai-sdk/openai"; +import { createNoThinkFetch, type DisableThinkingStrategy } from "../../utils/no-think-fetch.js"; import type { PluginLogger } from "../types.js"; const TAG = "[context-offload] [local-llm]"; @@ -16,6 +17,11 @@ export interface LlmCallerConfig { model: string; temperature: number; timeoutMs: number; + /** + * Controls how thinking/reasoning is disabled for the LLM endpoint. + * See DisableThinkingStrategy for the full list of strategies. + */ + disableThinking?: DisableThinkingStrategy; } export interface CallLlmOpts { @@ -27,6 +33,8 @@ export interface CallLlmOpts { timeoutMs?: number; /** Label for logging (e.g. "L1", "L1.5", "L2") */ label?: string; + /** Pre-created fetch wrapper (for caching at the client level). */ + customFetch?: typeof globalThis.fetch; } /** @@ -48,10 +56,15 @@ export async function callLlm( `systemLen=${opts.systemPrompt.length}, userLen=${opts.userPrompt.length}`, ); + const customFetch = opts.customFetch ?? ( + config.disableThinking ? createNoThinkFetch(config.disableThinking) : undefined + ); + const provider = createOpenAI({ baseURL: config.baseUrl, apiKey: config.apiKey, compatibility: "compatible", + ...(customFetch ? { fetch: customFetch } : {}), }); try { diff --git a/src/utils/no-think-fetch.test.ts b/src/utils/no-think-fetch.test.ts new file mode 100644 index 0000000..ad52231 --- /dev/null +++ b/src/utils/no-think-fetch.test.ts @@ -0,0 +1,269 @@ +import { describe, it, expect, vi, afterEach } from "vitest"; +import { + createNoThinkFetch, + isValidDisableThinkingStrategy, + normalizeDisableThinking, + type DisableThinkingStrategy, +} from "./no-think-fetch"; + +/** + * Capture the (input, init) passed through to the real global fetch so we can + * assert on the (possibly rewritten) request body. The mock never blindly + * JSON.parses the body — it just records and returns a stub Response. + */ +function captureFetch() { + const calls: Array<{ input: unknown; init: RequestInit | undefined }> = []; + vi.spyOn(globalThis, "fetch").mockImplementation((async (input: unknown, init?: RequestInit) => { + calls.push({ input, init }); + return new Response("{}", { status: 200 }); + }) as typeof globalThis.fetch); + return calls; +} + +afterEach(() => { + vi.restoreAllMocks(); +}); + +describe("createNoThinkFetch", () => { + // ─── vllm strategy (original behavior) ──────────────────────────────────── + + describe("vllm strategy", () => { + it("injects chat_template_kwargs.enable_thinking=false into chat bodies", async () => { + const calls = captureFetch(); + const f = createNoThinkFetch("vllm"); + + await f("https://example/v1/chat/completions", { + method: "POST", + body: JSON.stringify({ model: "qwen3", messages: [{ role: "user", content: "hi" }] }), + }); + + const sent = JSON.parse(calls[0].init!.body as string); + expect(sent.chat_template_kwargs).toEqual({ enable_thinking: false }); + expect(sent.messages).toHaveLength(1); + }); + + it("merges into an existing chat_template_kwargs instead of clobbering it", async () => { + const calls = captureFetch(); + const f = createNoThinkFetch("vllm"); + + await f("https://example", { + body: JSON.stringify({ messages: [], chat_template_kwargs: { foo: "bar", enable_thinking: true } }), + }); + + const sent = JSON.parse(calls[0].init!.body as string); + expect(sent.chat_template_kwargs).toEqual({ foo: "bar", enable_thinking: false }); + }); + + it("leaves embedding requests (input, no messages) untouched", async () => { + const calls = captureFetch(); + const f = createNoThinkFetch("vllm"); + const body = JSON.stringify({ model: "bge-m3", input: ["hello"] }); + + await f("https://example/v1/embeddings", { body }); + + expect(calls[0].init!.body).toBe(body); + expect(JSON.parse(calls[0].init!.body as string).chat_template_kwargs).toBeUndefined(); + }); + }); + + // ─── deepseek strategy ─────────────────────────────────────────────────── + + describe("deepseek strategy", () => { + it("injects top-level enable_thinking: false", async () => { + const calls = captureFetch(); + const f = createNoThinkFetch("deepseek"); + + await f("https://api.deepseek.com/v1/chat/completions", { + method: "POST", + body: JSON.stringify({ model: "deepseek-reasoner", messages: [{ role: "user", content: "hi" }] }), + }); + + const sent = JSON.parse(calls[0].init!.body as string); + expect(sent.enable_thinking).toBe(false); + expect(sent.chat_template_kwargs).toBeUndefined(); + expect(sent.messages).toHaveLength(1); + }); + }); + + // ─── dashscope strategy ───────────────────────────────────────────────── + + describe("dashscope strategy", () => { + it("injects top-level enable_thinking: false", async () => { + const calls = captureFetch(); + const f = createNoThinkFetch("dashscope"); + + await f("https://dashscope.aliyuncs.com/compatible-mode/v1/chat/completions", { + method: "POST", + body: JSON.stringify({ model: "qwen-plus", messages: [{ role: "user", content: "hi" }] }), + }); + + const sent = JSON.parse(calls[0].init!.body as string); + expect(sent.enable_thinking).toBe(false); + expect(sent.chat_template_kwargs).toBeUndefined(); + }); + }); + + // ─── openai strategy ──────────────────────────────────────────────────── + + describe("openai strategy", () => { + it("injects reasoning_effort: low", async () => { + const calls = captureFetch(); + const f = createNoThinkFetch("openai"); + + await f("https://api.openai.com/v1/chat/completions", { + method: "POST", + body: JSON.stringify({ model: "o3-mini", messages: [{ role: "user", content: "hi" }] }), + }); + + const sent = JSON.parse(calls[0].init!.body as string); + expect(sent.reasoning_effort).toBe("low"); + expect(sent.chat_template_kwargs).toBeUndefined(); + expect(sent.enable_thinking).toBeUndefined(); + }); + }); + + // ─── anthropic strategy ───────────────────────────────────────────────── + + describe("anthropic strategy", () => { + it("injects thinking: { type: disabled }", async () => { + const calls = captureFetch(); + const f = createNoThinkFetch("anthropic"); + + await f("https://api.anthropic.com/v1/messages", { + method: "POST", + body: JSON.stringify({ model: "claude-sonnet-4-20250514", messages: [{ role: "user", content: "hi" }] }), + }); + + const sent = JSON.parse(calls[0].init!.body as string); + expect(sent.thinking).toEqual({ type: "disabled" }); + expect(sent.chat_template_kwargs).toBeUndefined(); + }); + }); + + // ─── kimi strategy ───────────────────────────────────────────────────── + + describe("kimi strategy", () => { + it("injects thinking: { type: disabled }", async () => { + const calls = captureFetch(); + const f = createNoThinkFetch("kimi"); + + await f("https://api.moonshot.cn/v1/chat/completions", { + method: "POST", + body: JSON.stringify({ model: "kimi-k2.6", messages: [{ role: "user", content: "hi" }] }), + }); + + const sent = JSON.parse(calls[0].init!.body as string); + expect(sent.thinking).toEqual({ type: "disabled" }); + expect(sent.chat_template_kwargs).toBeUndefined(); + expect(sent.enable_thinking).toBeUndefined(); + }); + }); + + // ─── gemini strategy ──────────────────────────────────────────────────── + + describe("gemini strategy", () => { + it("injects thinking_config: { thinking_budget: 0 }", async () => { + const calls = captureFetch(); + const f = createNoThinkFetch("gemini"); + + await f("https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash:generateContent", { + method: "POST", + body: JSON.stringify({ messages: [{ role: "user", content: "hi" }] }), + }); + + const sent = JSON.parse(calls[0].init!.body as string); + expect(sent.thinking_config).toEqual({ thinking_budget: 0 }); + expect(sent.chat_template_kwargs).toBeUndefined(); + }); + }); + + // ─── strategy === false (passthrough) ─────────────────────────────────── + + describe("false strategy", () => { + it("returns globalThis.fetch directly", () => { + const f = createNoThinkFetch(false); + expect(f).toBe(globalThis.fetch); + }); + + it("default parameter returns globalThis.fetch", () => { + const f = createNoThinkFetch(); + expect(f).toBe(globalThis.fetch); + }); + }); + + // ─── Common behavior across all strategies ────────────────────────────── + + describe("common behavior", () => { + it("forwards a non-JSON string body unchanged", async () => { + const calls = captureFetch(); + const f = createNoThinkFetch("vllm"); + + await f("https://example", { body: "not-json" }); + + expect(calls[0].init!.body).toBe("not-json"); + }); + + it("forwards requests with no init and with a non-string body unchanged", async () => { + const calls = captureFetch(); + const f = createNoThinkFetch("deepseek"); + + await f("https://example"); + expect(calls[0].init).toBeUndefined(); + + const blob = new Uint8Array([1, 2, 3]); + await f("https://example", { body: blob }); + expect(calls[1].init!.body).toBe(blob); + }); + }); +}); + +// ─── Validation helpers ───────────────────────────────────────────────────── + +describe("isValidDisableThinkingStrategy", () => { + it("returns true for all valid strategies", () => { + const valid: DisableThinkingStrategy[] = [false, "vllm", "deepseek", "dashscope", "openai", "anthropic", "kimi", "gemini"]; + for (const v of valid) { + expect(isValidDisableThinkingStrategy(v)).toBe(true); + } + }); + + it("returns false for invalid values", () => { + const invalid = [true, "invalid", "sglang", "VLLM", 0, null, undefined, "", "true"]; + for (const v of invalid) { + expect(isValidDisableThinkingStrategy(v)).toBe(false); + } + }); +}); + +describe("normalizeDisableThinking", () => { + it("maps true to vllm (shorthand)", () => { + expect(normalizeDisableThinking(true)).toBe("vllm"); + }); + + it("maps false to false", () => { + expect(normalizeDisableThinking(false)).toBe(false); + }); + + it("maps undefined to false", () => { + expect(normalizeDisableThinking(undefined)).toBe(false); + }); + + it("passes through valid strategy strings", () => { + expect(normalizeDisableThinking("vllm")).toBe("vllm"); + expect(normalizeDisableThinking("deepseek")).toBe("deepseek"); + expect(normalizeDisableThinking("dashscope")).toBe("dashscope"); + expect(normalizeDisableThinking("openai")).toBe("openai"); + expect(normalizeDisableThinking("anthropic")).toBe("anthropic"); + expect(normalizeDisableThinking("kimi")).toBe("kimi"); + expect(normalizeDisableThinking("gemini")).toBe("gemini"); + }); + + it("warns and returns false for unknown strategies", () => { + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + expect(normalizeDisableThinking("unknown_strategy")).toBe(false); + expect(warnSpy).toHaveBeenCalledWith( + expect.stringContaining('Unknown disableThinking strategy "unknown_strategy"'), + ); + warnSpy.mockRestore(); + }); +}); diff --git a/src/utils/no-think-fetch.ts b/src/utils/no-think-fetch.ts new file mode 100644 index 0000000..6f68c2c --- /dev/null +++ b/src/utils/no-think-fetch.ts @@ -0,0 +1,135 @@ +/** + * Multi-strategy fetch wrapper for disabling thinking/reasoning across + * different inference engines and model providers. + * + * Each strategy injects provider-specific fields into chat-completion + * request bodies. Non-chat requests (embeddings, etc.) pass through + * unchanged. + * + * Strategies: + * - `"vllm"`: vLLM / SGLang — `chat_template_kwargs.enable_thinking = false` + * - `"deepseek"`: DeepSeek official API — top-level `enable_thinking: false` + * - `"dashscope"`: Alibaba DashScope (Qwen) — top-level `enable_thinking: false` + * - `"openai"`: OpenAI o-series — `reasoning_effort: "low"` (cannot fully disable) + * - `"anthropic"`: Anthropic Claude — `thinking: { type: "disabled" }` + * - `"kimi"`: Kimi (Moonshot) — `thinking: { type: "disabled" }` + * - `"gemini"`: Google Gemini — `thinking_config: { thinking_budget: 0 }` + */ + +// ─── Type & validation ──────────────────────────────────────────────────────── + +export type DisableThinkingStrategy = + | false + | "vllm" + | "deepseek" + | "dashscope" + | "openai" + | "anthropic" + | "kimi" + | "gemini"; + +export const VALID_DISABLE_THINKING_STRATEGIES: readonly DisableThinkingStrategy[] = [ + false, "vllm", "deepseek", "dashscope", "openai", "anthropic", "kimi", "gemini", +] as const; + +/** Check if a value is a valid DisableThinkingStrategy. */ +export function isValidDisableThinkingStrategy(value: unknown): value is DisableThinkingStrategy { + return (VALID_DISABLE_THINKING_STRATEGIES as readonly unknown[]).includes(value); +} + +/** + * Normalize a raw boolean-or-string config value into a DisableThinkingStrategy. + * + * true → "vllm" (shorthand for the most common self-hosted scenario) + * false / undefined → false + * + * Unknown string values fall back to false with a console warning. + */ +export function normalizeDisableThinking(raw: boolean | string | undefined): DisableThinkingStrategy { + if (raw === undefined || raw === false) return false; + if (raw === true) return "vllm"; + // raw is a string + if (isValidDisableThinkingStrategy(raw)) return raw; + console.warn( + `[memory-tdai] Unknown disableThinking strategy "${raw}", ` + + `valid values: false, true, "vllm", "deepseek", "dashscope", "openai", "anthropic", "kimi", "gemini". ` + + `Thinking will NOT be disabled.`, + ); + return false; +} + +// ─── Per-provider body transformers ─────────────────────────────────────────── + +function applyVllm(body: Record): void { + const existing = body.chat_template_kwargs; + const base = (existing && typeof existing === "object" && !Array.isArray(existing)) + ? existing as Record + : {}; + body.chat_template_kwargs = { ...base, enable_thinking: false }; +} + +function applyDeepSeek(body: Record): void { + body.enable_thinking = false; +} + +function applyDashScope(body: Record): void { + body.enable_thinking = false; +} + +function applyOpenAI(body: Record): void { + body.reasoning_effort = "low"; +} + +function applyAnthropic(body: Record): void { + body.thinking = { type: "disabled" }; +} + +function applyGemini(body: Record): void { + body.thinking_config = { thinking_budget: 0 }; +} + +const STRATEGY_TRANSFORMERS: Record< + Exclude, + (body: Record) => void +> = { + vllm: applyVllm, + deepseek: applyDeepSeek, + dashscope: applyDashScope, + openai: applyOpenAI, + anthropic: applyAnthropic, + kimi: applyAnthropic, + gemini: applyGemini, +}; + +// ─── Factory ────────────────────────────────────────────────────────────────── + +/** + * Create a fetch wrapper that injects provider-specific thinking-disabling + * fields into chat-completion request bodies. + * + * When `strategy` is `false`, returns `globalThis.fetch` directly (no wrapper). + * + * Only requests with a `messages` array in the body are modified — embedding + * and other non-chat requests pass through unchanged. + */ +export function createNoThinkFetch(strategy: DisableThinkingStrategy = false): typeof globalThis.fetch { + if (strategy === false) return globalThis.fetch; + + const transform = STRATEGY_TRANSFORMERS[strategy]; + if (!transform) return globalThis.fetch; // defensive: unknown strategy → passthrough + + return (async (input, init) => { + if (init && typeof init.body === "string") { + try { + const body = JSON.parse(init.body); + if (body && Array.isArray(body.messages)) { + transform(body); + init = { ...init, body: JSON.stringify(body) }; + } + } catch { + // non-JSON body — forward unchanged + } + } + return globalThis.fetch(input, init); + }) as typeof globalThis.fetch; +}