From 7a5fce9f127619f7e7c2a294d4c7fc803c06a6e2 Mon Sep 17 00:00:00 2001 From: chrishuan Date: Thu, 9 Apr 2026 18:23:46 +0800 Subject: [PATCH] feat: init Agent-Memory --- .gitignore | 14 + CHANGELOG.md | 29 + LICENSE | 27 + index.ts | 1337 +++++++++++++++++ openclaw.plugin.json | 90 ++ package.json | 61 + src/config.ts | 419 ++++++ src/conversation/l0-recorder.ts | 579 ++++++++ src/hooks/auto-capture.ts | 270 ++++ src/hooks/auto-recall.ts | 762 ++++++++++ src/persona/persona-generator.ts | 205 +++ src/persona/persona-trigger.ts | 121 ++ src/prompts/l1-dedup.ts | 163 +++ src/prompts/l1-extraction.ts | 137 ++ src/prompts/persona-generation.ts | 172 +++ src/prompts/scene-extraction.ts | 228 +++ src/record/l1-dedup.ts | 382 +++++ src/record/l1-extractor.ts | 500 +++++++ src/record/l1-reader.ts | 218 +++ src/record/l1-writer.ts | 280 ++++ src/report/reporter.ts | 95 ++ src/scene/scene-extractor.ts | 416 ++++++ src/scene/scene-format.ts | 75 + src/scene/scene-index.ts | 96 ++ src/scene/scene-navigation.ts | 57 + src/store/embedding.ts | 598 ++++++++ src/store/vector-store.ts | 2225 +++++++++++++++++++++++++++++ src/tools/conversation-search.ts | 279 ++++ src/tools/memory-search.ts | 290 ++++ src/utils/backup.ts | 156 ++ src/utils/checkpoint.ts | 543 +++++++ src/utils/clean-context-runner.ts | 451 ++++++ src/utils/managed-timer.ts | 138 ++ src/utils/memory-cleaner.ts | 331 +++++ src/utils/pipeline-manager.ts | 1087 ++++++++++++++ src/utils/sanitize.ts | 398 ++++++ src/utils/serial-queue.ts | 120 ++ src/utils/session-filter.ts | 108 ++ src/utils/text-utils.ts | 31 + 39 files changed, 13488 insertions(+) create mode 100644 .gitignore create mode 100644 CHANGELOG.md create mode 100644 LICENSE create mode 100644 index.ts create mode 100644 openclaw.plugin.json create mode 100644 package.json create mode 100644 src/config.ts create mode 100644 src/conversation/l0-recorder.ts create mode 100644 src/hooks/auto-capture.ts create mode 100644 src/hooks/auto-recall.ts create mode 100644 src/persona/persona-generator.ts create mode 100644 src/persona/persona-trigger.ts create mode 100644 src/prompts/l1-dedup.ts create mode 100644 src/prompts/l1-extraction.ts create mode 100644 src/prompts/persona-generation.ts create mode 100644 src/prompts/scene-extraction.ts create mode 100644 src/record/l1-dedup.ts create mode 100644 src/record/l1-extractor.ts create mode 100644 src/record/l1-reader.ts create mode 100644 src/record/l1-writer.ts create mode 100644 src/report/reporter.ts create mode 100644 src/scene/scene-extractor.ts create mode 100644 src/scene/scene-format.ts create mode 100644 src/scene/scene-index.ts create mode 100644 src/scene/scene-navigation.ts create mode 100644 src/store/embedding.ts create mode 100644 src/store/vector-store.ts create mode 100644 src/tools/conversation-search.ts create mode 100644 src/tools/memory-search.ts create mode 100644 src/utils/backup.ts create mode 100644 src/utils/checkpoint.ts create mode 100644 src/utils/clean-context-runner.ts create mode 100644 src/utils/managed-timer.ts create mode 100644 src/utils/memory-cleaner.ts create mode 100644 src/utils/pipeline-manager.ts create mode 100644 src/utils/sanitize.ts create mode 100644 src/utils/serial-queue.ts create mode 100644 src/utils/session-filter.ts create mode 100644 src/utils/text-utils.ts diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9cde883 --- /dev/null +++ b/.gitignore @@ -0,0 +1,14 @@ +# Dependencies +node_modules/ + +# Runtime workspace +workspace/ + +# Environment variables +.env + +# Test caches +__tests__/soak/.model-cache/ + +node_modules/ +benchmark-runs/ \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..b5c8195 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,29 @@ +# Changelog + +本文件记录 `@tencentdb-agent-memory/memory-tencentdb` 插件的所有显著变更，格式遵循 [Keep a Changelog](https://keepachangelog.com/zh-CN/1.1.0/)，版本号遵循 [Semantic Versioning](https://semver.org/)。 + +--- + +## [0.1.2] — 2026-03-26 + +### 更新内容 + +1. 优化对话捕获与记忆抽取过滤机制 + +## [0.1.1] — 2026-03-25 + +### 更新内容 + +1. 兼容 openclaw 2026.3.23 更新 + +## [0.1.0] — 2026-03-25 + +> 首个正式发布版本。本地优先的四层记忆系统（L0→L1→L2→L3），基于 SQLite + LLM 实现对话捕获、记忆提取、场景归纳与用户画像。 + +### 更新内容 + +1. 关键字检索增加 FTS5 全文索引，采用 jieba 分词 +2. 未配置远程 embedding 服务时，默认不开启 embedding 能力（不自动使用本地 embedding，且封禁主动使用本地 embedding 的配置入口） +3. 优化 L2、L3 生成 prompt 以控制生成内容大小（减少 token 开销） +4. Pipeline 调度器优化文件锁用法 +5. 避免全量读取 L0、L1 数据 diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..86fd8a1 --- /dev/null +++ b/LICENSE @@ -0,0 +1,27 @@ +Tencent is pleased to support the open source community by making TencentDB Agent Memory available. + +Copyright (C) 2026 Tencent. All rights reserved. + +TencentDB Agent Memory is licensed under the MIT. + + +Terms of the MIT: +-------------------------------------------------------------------- +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/index.ts b/index.ts new file mode 100644 index 0000000..594fb8f --- /dev/null +++ b/index.ts @@ -0,0 +1,1337 @@ +/** + * memory-tdai v3: Four-layer memory system plugin for OpenClaw. + * + * Provides: + * - L0: Automatic conversation recording (local JSONL) + * - L1: Structured memory extraction (LLM + dedup) + * - L2: Scene block management (LLM scene extraction) + * - L3: Persona generation (LLM persona synthesis) + * + * All processing is local, zero external API dependencies. + */ + +import fs from "node:fs"; +import path from "node:path"; +import { createRequire } from "node:module"; +import type { OpenClawPluginApi } from "openclaw/plugin-sdk/core"; +import { parseConfig } from "./src/config.js"; +import type { MemoryTdaiConfig } from "./src/config.js"; +import { performAutoRecall } from "./src/hooks/auto-recall.js"; +import { performAutoCapture } from "./src/hooks/auto-capture.js"; +import { MemoryPipelineManager } from "./src/utils/pipeline-manager.js"; +import { SceneExtractor } from "./src/scene/scene-extractor.js"; +import { CheckpointManager } from "./src/utils/checkpoint.js"; +import { PersonaTrigger } from "./src/persona/persona-trigger.js"; +import { PersonaGenerator } from "./src/persona/persona-generator.js"; +import { prewarmEmbeddedAgent } from "./src/utils/clean-context-runner.js"; +import { SessionFilter } from "./src/utils/session-filter.js"; +import { extractL1Memories } from "./src/record/l1-extractor.js"; +import { readConversationMessagesGroupedBySessionId } from "./src/conversation/l0-recorder.js"; +import type { ConversationMessage } from "./src/conversation/l0-recorder.js"; +import { VectorStore } from "./src/store/vector-store.js"; +import { createEmbeddingService } from "./src/store/embedding.js"; +import type { EmbeddingService } from "./src/store/embedding.js"; +import { executeMemorySearch, formatSearchResponse } from "./src/tools/memory-search.js"; +import { executeConversationSearch, formatConversationSearchResponse } from "./src/tools/conversation-search.js"; +import { LocalMemoryCleaner } from "./src/utils/memory-cleaner.js"; +import { getOrCreateInstanceId, initReporter, report } from "./src/report/reporter.js"; + +const TAG = "[memory-tdai]"; + +/** + * Initialize all required data directories under the plugin data root. + * + * Called once at plugin registration time so downstream modules + * (L0 recorder, L1 writer, scene extractor, persona generator, etc.) + * don't need to lazily mkdir on every write — the directories are + * guaranteed to exist from startup. + * + * Directory layout: + * / + * ├── conversations/ — L0 daily JSONL shards (one message per line) + * ├── records/ — L1 daily JSONL shards (extracted memories) + * ├── scene_blocks/ — L2 scene block .md files (LLM-managed) + * ├── .metadata/ — checkpoint, scene_index.json + * └── .backup/ — rotating backups (persona, scene_blocks) + */ +function initDataDirectories(dataDir: string): void { + const dirs = [ + "conversations", + "records", + "scene_blocks", + ".metadata", + ".backup", + ]; + for (const sub of dirs) { + fs.mkdirSync(path.join(dataDir, sub), { recursive: true }); + } +} + +/** + * Epoch ms when the plugin was registered (cold-start timestamp). + * Used as a fallback cursor in performAutoCapture when no checkpoint + * exists yet — prevents the first agent_end from dumping the entire + * session history into L0. + */ +let pluginStartTimestamp = 0; + +/** + * Cache original user prompts and message counts across hooks. + * - text: clean user prompt before prependContext injection + * - ts: cache creation time (for TTL sweep) + * - messageCount: session message count at before_prompt_build time, + * used as fallback slice offset if timestamp cursor is unreliable + */ +const pendingOriginalPrompts = new Map(); +const PROMPT_CACHE_TTL_MS = 10 * 60 * 1000; // 10 minutes +const PROMPT_CACHE_MAX_SIZE = 10_000; // Hard limit to prevent unbounded growth in high-concurrency scenarios + +/** + * Cache recall results (L1 memories + L3 Persona) from before_prompt_build + * for retrieval at agent_end, enabling the agent_turn metric event. + * + * Keyed by sessionKey — same correlation pattern as pendingOriginalPrompts. + */ +const pendingRecallCache = new Map; + l3Persona: string | null; + strategy: string; + durationMs: number; + ts: number; +}>(); + +/** + * Cache recall completion timestamps per session. + * Used in agent_end to estimate LLM reasoning time: + * llmEstimatedMs ≈ agent_end_start - recall_end_ts + * Entries are cleaned up in agent_end after use; stale entries swept alongside prompt cache. + */ +const pendingRecallEndTimestamps = new Map(); + +// 进程级单例，避免同一进程重复启动清理器导致并发清理竞态 +let sharedMemoryCleaner: LocalMemoryCleaner | undefined; + +/** + * Sweep both pendingOriginalPrompts and pendingRecallCache for stale entries. + * Unified from the original sweepStalePromptCache() to cover both Maps + * with identical TTL + hard-cap logic. + */ +function sweepStaleCaches(): void { + const now = Date.now(); + // Clean pendingOriginalPrompts + for (const [key, entry] of pendingOriginalPrompts) { + if (now - entry.ts > PROMPT_CACHE_TTL_MS) { + pendingOriginalPrompts.delete(key); + pendingRecallEndTimestamps.delete(key); + } + } + // Clean pendingRecallCache + for (const [key, entry] of pendingRecallCache) { + if (now - entry.ts > PROMPT_CACHE_TTL_MS) { + pendingRecallCache.delete(key); + } + } + // Hard limit: evict oldest entries if either Map exceeds cap + if (pendingOriginalPrompts.size > PROMPT_CACHE_MAX_SIZE) { + const entries = [...pendingOriginalPrompts.entries()].sort((a, b) => a[1].ts - b[1].ts); + const toEvict = entries.slice(0, entries.length - PROMPT_CACHE_MAX_SIZE); + for (const [key] of toEvict) { + pendingOriginalPrompts.delete(key); + pendingRecallEndTimestamps.delete(key); + } + } + if (pendingRecallCache.size > PROMPT_CACHE_MAX_SIZE) { + const entries = [...pendingRecallCache.entries()].sort((a, b) => a[1].ts - b[1].ts); + const toEvict = entries.slice(0, entries.length - PROMPT_CACHE_MAX_SIZE); + for (const [key] of toEvict) { + pendingRecallCache.delete(key); + } + } +} + +export default function register(api: OpenClawPluginApi) { + pluginStartTimestamp = Date.now(); + const _require = createRequire(import.meta.url); + const pluginVersion = (() => { try { return (_require("./package.json") as { version?: string }).version ?? "unknown"; } catch { return "unknown"; } })(); + api.logger.info( + `${TAG} Registering plugin ... ` + + `startTimestamp=${pluginStartTimestamp} (${new Date(pluginStartTimestamp).toISOString()})`, + ); + + // Persistent instance ID for metric reporting (populated async below) + let instanceId: string | undefined; + + let cfg: MemoryTdaiConfig; + try { + cfg = parseConfig(api.pluginConfig as Record | undefined); + api.logger.info( + `${TAG} Config parsed: ` + + `capture=${cfg.capture.enabled}, ` + + `recall=${cfg.recall.enabled}(maxResults=${cfg.recall.maxResults}), ` + + `extraction=${cfg.extraction.enabled}(dedup=${cfg.extraction.enableDedup}, maxMem=${cfg.extraction.maxMemoriesPerSession}), ` + + `pipeline=(everyN=${cfg.pipeline.everyNConversations}, warmup=${cfg.pipeline.enableWarmup}, l1Idle=${cfg.pipeline.l1IdleTimeoutSeconds}s, l2DelayAfterL1=${cfg.pipeline.l2DelayAfterL1Seconds}s, l2Min=${cfg.pipeline.l2MinIntervalSeconds}s, l2Max=${cfg.pipeline.l2MaxIntervalSeconds}s, activeWindow=${cfg.pipeline.sessionActiveWindowHours}h), ` + + `persona(triggerEvery=${cfg.persona.triggerEveryN}, backupCount=${cfg.persona.backupCount}, sceneBackupCount=${cfg.persona.sceneBackupCount}), ` + + `memoryCleanup(enabled=${cfg.memoryCleanup.enabled}, retentionDays=${cfg.memoryCleanup.retentionDays ?? "(disabled)"}, cleanTime=${cfg.memoryCleanup.cleanTime})`, + ); + } catch (err) { + api.logger.error(`${TAG} Config parsing failed: ${err instanceof Error ? err.message : String(err)}`); + throw err; + } + + // If remote embedding config is incomplete, log a prominent error so the user knows + if (cfg.embedding.configError) { + api.logger.error(`${TAG} [EMBEDDING CONFIG ERROR] ${cfg.embedding.configError}`); + } + + // Resolve plugin data directory via runtime API (avoid importing internal paths directly) + const pluginDataDir = path.join(api.runtime.state.resolveStateDir(), "memory-tdai"); + initDataDirectories(pluginDataDir); + api.logger.info(`${TAG} Data dir: ${pluginDataDir} (all subdirectories initialized)`); + + // Unified session/agent filter: combines internal-session detection + user-configured excludeAgents + const sessionFilter = new SessionFilter(cfg.capture.excludeAgents); + if (cfg.capture.excludeAgents.length > 0) { + api.logger.info(`${TAG} Agent exclude patterns: ${cfg.capture.excludeAgents.join(", ")}`); + } + + // Daily local JSONL cleaner (L0/L1), enabled only when retentionDays is configured. + let memoryCleaner: LocalMemoryCleaner | undefined; + if (cfg.memoryCleanup.enabled && cfg.memoryCleanup.retentionDays != null) { + if (!sharedMemoryCleaner) { + sharedMemoryCleaner = new LocalMemoryCleaner({ + baseDir: pluginDataDir, + retentionDays: cfg.memoryCleanup.retentionDays, + cleanTime: cfg.memoryCleanup.cleanTime, + logger: api.logger, + }); + sharedMemoryCleaner.start(); + api.logger.info(`${TAG} Memory cleaner started (singleton)`); + } else { + api.logger.info(`${TAG} Memory cleaner already started in this process, reusing existing instance`); + } + memoryCleaner = sharedMemoryCleaner; + } else { + api.logger.info(`${TAG} Memory cleaner disabled (retentionDays not configured)`); + } + + // Hardcoded actor ID (legacy, to be removed) + const ACTOR_ID = "default_user"; + + const resolveSessionKey = (sessionKey?: string): string | undefined => { + if (sessionKey) return sessionKey; + api.logger.warn(`${TAG} sessionKey is empty, skipping capture/recall to avoid unstable fallback key`); + return undefined; + }; + + // ============================ + // Tool registration + // ============================ + + // Shared references for tools (populated when extraction scheduler creates them) + let sharedVectorStore: VectorStore | undefined; + let sharedEmbeddingService: EmbeddingService | undefined; + + /** + * Whether the local embedding service warmup has been triggered at least once. + * Tracked separately from schedulerStarted because warmup should also + * be triggered from before_prompt_build (recall), not only agent_end. + */ + let embeddingWarmupTriggered = false; + + /** + * Trigger local embedding model warmup (download + load) on first use. + * Safe to call multiple times — delegates idempotency to startWarmup() itself. + * + * IMPORTANT: If a previous warmup attempt FAILED (e.g. model download + * network error), this will re-trigger startWarmup() so the service can + * retry. startWarmup() internally checks its state machine: + * - "ready" / "initializing" → no-op (already done or in progress) + * - "idle" / "failed" → starts a new initialization attempt + * + * This avoids triggering model download during short-lived CLI commands + * like `gateway stop` or `agents list` (warmup is still deferred until + * the first real conversation). + */ + const ensureEmbeddingWarmup = (): void => { + if (!sharedEmbeddingService) return; + + if (!embeddingWarmupTriggered) { + embeddingWarmupTriggered = true; + api.logger.debug?.(`${TAG} Triggering lazy embedding warmup on first conversation`); + sharedEmbeddingService.startWarmup(); + return; + } + + // After first trigger: re-invoke startWarmup() only if the service + // is not yet ready (covers the "failed" → retry path). + // startWarmup() is idempotent for "ready" and "initializing" states. + if (!sharedEmbeddingService.isReady()) { + api.logger.debug?.(`${TAG} Embedding not ready, re-triggering warmup (retry)`); + sharedEmbeddingService.startWarmup(); + } + }; + + // tdai_memory_search — Agent-callable L1 memory search tool + api.registerTool( + { + name: "tdai_memory_search", + label: "Memory Search", + description: + "Search through the user's long-term memories. Use this when you need to recall specific information about the user's preferences, past events, instructions, or context from previous conversations. Returns relevant memory records ranked by relevance.", + parameters: { + type: "object", + properties: { + query: { + type: "string", + description: "Search query describing what you want to recall about the user", + }, + limit: { + type: "number", + description: "Maximum number of results to return (default: 5, max: 20)", + }, + type: { + type: "string", + enum: ["persona", "episodic", "instruction"], + description: "Optional filter by memory type: persona (identity/preferences), episodic (events/activities), instruction (user rules/commands)", + }, + scene: { + type: "string", + description: "Optional filter by scene name", + }, + }, + required: ["query"], + }, + async execute(_toolCallId: string, params: Record) { + const startMs = Date.now(); + const query = String(params.query ?? ""); + const limit = Math.min(Math.max(Number(params.limit) || 5, 1), 20); + const typeFilter = typeof params.type === "string" ? params.type : undefined; + const sceneFilter = typeof params.scene === "string" ? params.scene : undefined; + + api.logger.debug?.( + `${TAG} [tool] tdai_memory_search called: ` + + `query="${query.length > 80 ? query.slice(0, 80) + "…" : query}", ` + + `limit=${limit}, type=${typeFilter ?? "(all)"}, scene=${sceneFilter ?? "(all)"}`, + ); + + try { + const result = await executeMemorySearch({ + query, + limit, + type: typeFilter, + scene: sceneFilter, + vectorStore: sharedVectorStore, + embeddingService: sharedEmbeddingService, + logger: api.logger, + }); + + const elapsedMs = Date.now() - startMs; + const responseText = formatSearchResponse(result); + api.logger.debug?.( + `${TAG} [tool] tdai_memory_search completed (${elapsedMs}ms): ` + + `total=${result.total}, strategy=${result.strategy}, ` + + `responseLength=${responseText.length} chars`, + ); + report("tool_call", { + tool: "tdai_memory_search", + query, limit, typeFilter, sceneFilter, + resultCount: result.total, + strategy: result.strategy, + results: result.results, + durationMs: elapsedMs, + success: true, + }); + return { + content: [{ type: "text" as const, text: responseText }], + details: { count: result.total, strategy: result.strategy }, + }; + } catch (err) { + const elapsedMs = Date.now() - startMs; + const errMsg = err instanceof Error ? err.message : String(err); + api.logger.error(`${TAG} [tool] tdai_memory_search failed (${elapsedMs}ms): ${errMsg}`); + report("tool_call", { + tool: "tdai_memory_search", + query, limit, typeFilter, sceneFilter, + durationMs: elapsedMs, + success: false, + error: errMsg, + }); + return { + content: [{ type: "text" as const, text: `Memory search failed: ${errMsg}` }], + details: { error: errMsg }, + }; + } + }, + }, + { name: "tdai_memory_search" }, + ); + + // tdai_conversation_search — Agent-callable L0 conversation search tool + api.registerTool( + { + name: "tdai_conversation_search", + label: "Conversation Search", + description: + "Search through past conversation history (raw dialogue records). " + + "Use this when tdai_memory_search (structured memories) doesn't have the information you need, " + + "or when you want to find specific past conversations, dialogue context, or exact words " + + "the user said before. Returns relevant individual messages ranked by relevance.", + parameters: { + type: "object", + properties: { + query: { + type: "string", + description: "Search query describing what conversation content you want to find", + }, + limit: { + type: "number", + description: "Maximum number of messages to return (default: 5, max: 20)", + }, + session_key: { + type: "string", + description: "Optional: filter results to a specific session", + }, + }, + required: ["query"], + }, + async execute(_toolCallId: string, params: Record) { + const startMs = Date.now(); + const query = String(params.query ?? ""); + const limit = Math.min(Math.max(Number(params.limit) || 5, 1), 20); + const sessionKeyFilter = typeof params.session_key === "string" ? params.session_key : undefined; + + api.logger.debug?.( + `${TAG} [tool] tdai_conversation_search called: ` + + `query="${query.length > 80 ? query.slice(0, 80) + "…" : query}", ` + + `limit=${limit}, session_key=${sessionKeyFilter ?? "(all)"}`, + ); + + try { + const result = await executeConversationSearch({ + query, + limit, + sessionKey: sessionKeyFilter, + vectorStore: sharedVectorStore, + embeddingService: sharedEmbeddingService, + logger: api.logger, + }); + + const elapsedMs = Date.now() - startMs; + const responseText = formatConversationSearchResponse(result); + api.logger.debug?.( + `${TAG} [tool] tdai_conversation_search completed (${elapsedMs}ms): ` + + `total=${result.total}, responseLength=${responseText.length} chars`, + ); + report("tool_call", { + tool: "tdai_conversation_search", + query, limit, sessionKeyFilter, + resultCount: result.total, + strategy: result.strategy, + results: result.results, + durationMs: elapsedMs, + success: true, + }); + return { + content: [{ type: "text" as const, text: responseText }], + details: { count: result.total }, + }; + } catch (err) { + const elapsedMs = Date.now() - startMs; + const errMsg = err instanceof Error ? err.message : String(err); + api.logger.error(`${TAG} [tool] tdai_conversation_search failed (${elapsedMs}ms): ${errMsg}`); + report("tool_call", { + tool: "tdai_conversation_search", + query, limit, sessionKeyFilter, + durationMs: elapsedMs, + success: false, + error: errMsg, + }); + return { + content: [{ type: "text" as const, text: `Conversation search failed: ${errMsg}` }], + details: { error: errMsg }, + }; + } + }, + }, + { name: "tdai_conversation_search" }, + ); + + // ============================ + // Lifecycle hooks + // ============================ + + // Before prompt build: auto-recall relevant memories + // (migrated from legacy before_agent_start to before_prompt_build so that + // event.messages is guaranteed to be available — session is already loaded) + if (cfg.recall.enabled) { + api.logger.info(`${TAG} Registering before_prompt_build hook (auto-recall)`); + api.on("before_prompt_build", async (event, ctx) => { + const startMs = Date.now(); + api.logger.debug?.(`${TAG} [before_prompt_build] Hook triggered`); + + const sessionKey = ctx.sessionKey; + + if (sessionFilter.shouldSkipCtx(ctx)) { + api.logger.debug?.(`${TAG} [before_prompt_build] Skipping filtered session`); + return; + } + + // Trigger embedding warmup on first real conversation (lazy init). + // This is the earliest point where a real user message arrives, + // so we start the model download here rather than in register() + // to avoid triggering it during short-lived CLI commands. + ensureEmbeddingWarmup(); + + // Cache original user prompt for agent_end + const rawPrompt = event.prompt; + const messages = Array.isArray(event.messages) ? event.messages : undefined; + if (sessionKey && rawPrompt) { + const messageCount = messages?.length ?? 0; + pendingOriginalPrompts.set(sessionKey, { text: rawPrompt, ts: Date.now(), messageCount }); + api.logger.debug?.(`${TAG} [before_prompt_build] Cached original prompt (${rawPrompt.length} chars, msgCount=${messageCount})`); + } + sweepStaleCaches(); + + const userText = rawPrompt; + api.logger.debug?.(`${TAG} [before_prompt_build] userText length: ${userText?.length}`); + if (!userText) { + api.logger.debug?.(`${TAG} [before_prompt_build] No user text found, skipping recall`); + return; + } + + const resolvedSessionKey = resolveSessionKey(sessionKey); + if (!resolvedSessionKey) { + return; + } + + try { + const recallStartMs = Date.now(); + const result = await performAutoRecall({ + userText, + actorId: ACTOR_ID, + sessionKey: resolvedSessionKey, + cfg, + pluginDataDir, + logger: api.logger, + vectorStore: sharedVectorStore, + embeddingService: sharedEmbeddingService, + }); + const elapsedMs = Date.now() - startMs; + const recallDurationMs = Date.now() - recallStartMs; + + // Cache recall results for agent_turn metric (retrieved at agent_end) + if (sessionKey && result) { + pendingRecallCache.set(sessionKey, { + l1Memories: result.recalledL1Memories ?? [], + l3Persona: result.recalledL3Persona ?? null, + strategy: result.recallStrategy ?? "unknown", + durationMs: recallDurationMs, + ts: Date.now(), + }); + } + + // Record recall completion timestamp for LLM timing estimation in agent_end + if (resolvedSessionKey) { + pendingRecallEndTimestamps.set(resolvedSessionKey, Date.now()); + } + + if (result?.appendSystemContext) { + api.logger.info( + `${TAG} [before_prompt_build] Recall complete (${elapsedMs}ms), ` + + `appendSystemContext=${result.appendSystemContext.length} chars`, + ); + } else { + api.logger.info(`${TAG} [before_prompt_build] Recall complete (${elapsedMs}ms), no context to inject`); + } + return result; + } catch (err) { + const elapsedMs = Date.now() - startMs; + api.logger.error(`${TAG} [before_prompt_build] Auto-recall failed after ${elapsedMs}ms: ${err instanceof Error ? err.stack ?? err.message : String(err)}`); + // ── error_degradation metric ── + if (instanceId) { + report("error_degradation", { + module: "auto-recall", + action: "performAutoRecall", + errorType: "exception", + errorMessage: err instanceof Error ? err.message : String(err), + degradedTo: "no_recall", + impact: "non-blocking", + }); + } + } + }); + } + + // After agent end: auto-capture + L0 record + L1/L2/L3 schedule + if (cfg.capture.enabled) { + // ============================ + // Create the MemoryPipelineManager (L1→L2→L3 architecture) + // ============================ + let scheduler: MemoryPipelineManager | undefined; + + // ============================ + // Lazy scheduler startup (Solution C): + // Defer scheduler.start() until the first agent_end event. This way, + // short-lived CLI management commands (agents add/list/delete, etc.) + // never start the scheduler, never recover pending sessions, and + // therefore never trigger the L1→L2→L3 flush chain on destroy(). + // ============================ + let schedulerStarted = false; + + /** + * Lazily start the scheduler on first conversation. + * Reads checkpoint, restores session states, and pre-warms the + * embedded agent. Subsequent calls are no-ops. + * No-op when scheduler is undefined (extraction disabled). + */ + const ensureSchedulerStarted = async (): Promise => { + if (schedulerStarted || !scheduler) return; + schedulerStarted = true; + + // Propagate instanceId to scheduler for pipeline metrics + if (instanceId) { + scheduler.instanceId = instanceId; + } + + // Trigger embedding warmup alongside scheduler start — both are + // deferred until the first real conversation to avoid downloading + // models during short-lived CLI commands. + ensureEmbeddingWarmup(); + + try { + const initCheckpoint = new CheckpointManager(pluginDataDir, api.logger); + const cp = await initCheckpoint.read(); + scheduler.start(initCheckpoint.getAllPipelineStates(cp)); + api.logger.info( + `${TAG} Scheduler lazy-started on first agent_end ` + + `(everyN=${cfg.pipeline.everyNConversations}, ` + + `l1Idle=${cfg.pipeline.l1IdleTimeoutSeconds}s, ` + + `l2DelayAfterL1=${cfg.pipeline.l2DelayAfterL1Seconds}s, ` + + `l2MinInterval=${cfg.pipeline.l2MinIntervalSeconds}s, ` + + `l2MaxInterval=${cfg.pipeline.l2MaxIntervalSeconds}s, ` + + `sessionActiveWindow=${cfg.pipeline.sessionActiveWindowHours}h)`, + ); + } catch (err) { + api.logger.error( + `${TAG} Failed to restore checkpoint for scheduler: ${err instanceof Error ? err.message : String(err)}`, + ); + // Start with empty state as fallback + scheduler.start({}); + } + + // Pre-warm the embedded agent import so the first extraction run doesn't + // pay the cold-start cost (~35s jiti compile → <50ms with dist/ path). + prewarmEmbeddedAgent(api.logger); + }; + + if (cfg.extraction.enabled) { + // === Initialize VectorStore (always) + EmbeddingService (only when embedding enabled) === + let vectorStore: VectorStore | undefined; + let embeddingService: EmbeddingService | undefined; + + // VectorStore is always created as the metadata store for L0/L1 records. + // It works as a pure SQLite store even without embedding — keyword search, + // L0/L1 reads, and pipeline queries all use structured SQL, not vectors. + try { + const dims = cfg.embedding.dimensions; // 0 when provider="none" → vec0 tables deferred + const dbPath = path.join(pluginDataDir, "vectors.db"); + vectorStore = new VectorStore(dbPath, dims, api.logger); + + // Create EmbeddingService only when embedding is enabled (remote provider configured) + if (cfg.embedding.enabled) { + try { + if (cfg.embedding.provider !== "local" && cfg.embedding.apiKey) { + // Remote embedding provider (OpenAI-compatible API: OpenAI, Azure, self-hosted, etc.) + embeddingService = createEmbeddingService({ + provider: cfg.embedding.provider, + baseUrl: cfg.embedding.baseUrl, + apiKey: cfg.embedding.apiKey, + model: cfg.embedding.model, + dimensions: cfg.embedding.dimensions, + proxyUrl: cfg.embedding.proxyUrl, + maxInputChars: cfg.embedding.maxInputChars, + timeoutMs: cfg.embedding.timeoutMs, + }, api.logger); + } else { + // Local provider (node-llama-cpp) — preserved internally but not reachable from user config + embeddingService = createEmbeddingService({ + provider: "local", + modelPath: cfg.embedding.model || undefined, + modelCacheDir: cfg.embedding.modelCacheDir, + }, api.logger); + } + } catch (err) { + api.logger.warn( + `${TAG} EmbeddingService init failed, continuing with keyword-only mode: ${err instanceof Error ? err.message : String(err)}`, + ); + embeddingService = undefined; + } + } else { + api.logger.info(`${TAG} Embedding disabled by config, VectorStore will serve as metadata-only store`); + } + + // Init VectorStore with provider info (undefined when no embedding → skips provider change detection) + const providerInfo = embeddingService?.getProviderInfo(); + const initResult = vectorStore.init(providerInfo); + + // If VectorStore entered degraded mode (e.g. sqlite-vec load failed), + // treat it as unavailable and fall back to keyword-only mode. + if (vectorStore.isDegraded()) { + api.logger.warn( + `${TAG} VectorStore is in degraded mode, falling back to keyword dedup`, + ); + vectorStore = undefined; + embeddingService = undefined; + } else { + api.logger.info( + `${TAG} VectorStore initialized: ${dbPath} (${dims}D, provider=${cfg.embedding.provider})`, + ); + + // If embedding provider/model/dimensions changed, re-embed all existing texts + if (initResult.needsReindex && embeddingService) { + const svc = embeddingService; // capture for async closure + const vs = vectorStore; // capture for async closure + api.logger.info( + `${TAG} Embedding config changed (${initResult.reason}). ` + + `Starting background re-embed of all stored texts...`, + ); + // Run re-embed asynchronously so it doesn't block plugin startup + vs.reindexAll( + (text) => svc.embed(text), + (done, total, layer) => { + if (done === total || done % 50 === 0) { + api.logger.debug?.(`${TAG} Re-embed progress: ${layer} ${done}/${total}`); + } + }, + ).then(({ l1Count, l0Count }) => { + api.logger.info( + `${TAG} Re-embed complete: L1=${l1Count} records, L0=${l0Count} messages`, + ); + }).catch((err) => { + api.logger.error( + `${TAG} Re-embed failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`, + ); + }); + } + } + } catch (err) { + api.logger.warn( + `${TAG} VectorStore init failed; vector/FTS recall and dedup conflict detection will be unavailable: ${err instanceof Error ? err.message : String(err)}`, + ); + vectorStore = undefined; + embeddingService = undefined; + } + + // Share vectorStore/embeddingService with tdai_memory_search tool + sharedVectorStore = vectorStore; + sharedEmbeddingService = embeddingService; + + // Keep cleaner's SQLite handle updated (singleton cleaner may start earlier). + memoryCleaner?.setVectorStore(vectorStore); + + // === Create pipeline manager === + scheduler = new MemoryPipelineManager( + { + everyNConversations: cfg.pipeline.everyNConversations, + enableWarmup: cfg.pipeline.enableWarmup, + l1: { idleTimeoutSeconds: cfg.pipeline.l1IdleTimeoutSeconds }, + l2: { + delayAfterL1Seconds: cfg.pipeline.l2DelayAfterL1Seconds, + minIntervalSeconds: cfg.pipeline.l2MinIntervalSeconds, + maxIntervalSeconds: cfg.pipeline.l2MaxIntervalSeconds, + sessionActiveWindowHours: cfg.pipeline.sessionActiveWindowHours, + }, + }, + api.logger, + sessionFilter, + ); + + // L1 runner: read L0 from DB (primary) or JSONL (fallback) → local LLM extraction → L1 JSONL + VectorStore + scheduler.setL1Runner(async ({ sessionKey }) => { + // L1 reads L0 data from VectorStore DB (primary, indexed query). + // Fallback: read from L0 JSONL files when VectorStore is unavailable. + if (!api.config) { + api.logger.debug?.(`${TAG} [pipeline-l1] No OpenClaw config, skipping L1 extraction`); + return { processedCount: 0 }; + } + + const checkpoint = new CheckpointManager(pluginDataDir, api.logger); + const cp = await checkpoint.read(); + const runnerState = checkpoint.getRunnerState(cp, sessionKey); + + api.logger.info( + `${TAG} [pipeline-l1] Session ${sessionKey}: ` + + `l1_cursor=${runnerState.last_l1_cursor || "(start)"}`, + ); + + try { + // Read L0 messages since last L1 cursor, grouped by sessionId. + // Within the same sessionKey, different sessionIds represent different + // conversation instances (e.g. after /reset). Each group is extracted + // independently so its sessionId is correctly associated with L1 records. + // + // Primary path: read from VectorStore DB (indexed query, fast). + // Fallback: read from L0 JSONL files (scan + parse, slower). + let groups: Array<{ sessionId: string; messages: ConversationMessage[] }>; + + if (vectorStore && !vectorStore.isDegraded()) { + // DB path: fast indexed query + // NOTE: When last_l1_cursor is 0 (first L1 run), we pass undefined + // to query all messages — but only those captured AFTER plugin start + // (L0 capture uses pluginStartTimestamp as floor, so DB won't contain + // pre-existing messages). This is safe because auto-capture already + // filters out messages older than pluginStartTimestamp. + const l1Cursor = runnerState.last_l1_cursor > 0 + ? runnerState.last_l1_cursor + : undefined; + const dbGroups = vectorStore.queryL0GroupedBySessionId( + sessionKey, + l1Cursor, + ); + // Cast role from string to "user" | "assistant" (DB stores as string) + groups = dbGroups.map((g) => ({ + sessionId: g.sessionId, + messages: g.messages.map((m) => ({ + id: m.id, + role: m.role as "user" | "assistant", + content: m.content, + timestamp: m.timestamp, + })), + })); + api.logger.debug?.( + `${TAG} [pipeline-l1] L0 data source: VectorStore DB`, + ); + } else { + // Fallback: JSONL files + api.logger.debug?.( + `${TAG} [pipeline-l1] L0 data source: JSONL files (VectorStore unavailable)`, + ); + const jsonlGroups = await readConversationMessagesGroupedBySessionId( + sessionKey, + pluginDataDir, + runnerState.last_l1_cursor || undefined, + api.logger, + 50, // Match DB path limit (queryL0ForL1 default) + ); + // Convert SessionIdMessageGroup[] to the same shape + groups = jsonlGroups.map((g) => ({ + sessionId: g.sessionId, + messages: g.messages, + })); + } + + if (groups.length === 0) { + api.logger.debug?.(`${TAG} [pipeline-l1] No new L0 messages for session ${sessionKey}`); + return { processedCount: 0 }; + } + + const totalMessages = groups.reduce((sum, g) => sum + g.messages.length, 0); + api.logger.info( + `${TAG} [pipeline-l1] Processing ${totalMessages} L0 messages across ${groups.length} sessionId group(s) for session ${sessionKey}`, + ); + + let totalExtracted = 0; + let totalStored = 0; + let lastSceneName: string | undefined; + let maxTimestamp = 0; + + for (const group of groups) { + api.logger.debug?.( + `${TAG} [pipeline-l1] Group sessionId=${group.sessionId || "(empty)"}: ${group.messages.length} messages`, + ); + + const l1Result = await extractL1Memories({ + messages: group.messages, + sessionKey, + sessionId: group.sessionId, + baseDir: pluginDataDir, + config: api.config, + options: { + enableDedup: cfg.extraction.enableDedup, + maxMemoriesPerSession: cfg.extraction.maxMemoriesPerSession, + model: cfg.extraction.model, + previousSceneName: lastSceneName ?? (runnerState.last_scene_name || undefined), + vectorStore, + embeddingService, + conflictRecallTopK: cfg.embedding.conflictRecallTopK, + }, + logger: api.logger, + instanceId, + }); + + totalExtracted += l1Result.extractedCount; + totalStored += l1Result.storedCount; + if (l1Result.lastSceneName) { + lastSceneName = l1Result.lastSceneName; + } + + const groupMaxTs = Math.max(...group.messages.map((m) => m.timestamp)); + maxTimestamp = Math.max(maxTimestamp, groupMaxTs); + } + + // Update checkpoint on disk — cursor is the global max timestamp across all groups + await checkpoint.markL1ExtractionComplete( + sessionKey, + totalStored, + maxTimestamp, + lastSceneName, + ); + + api.logger.info( + `${TAG} [pipeline-l1] L1 complete: extracted=${totalExtracted}, stored=${totalStored} (${groups.length} group(s))`, + ); + + return { processedCount: totalMessages }; + } catch (err) { + api.logger.error(`${TAG} [pipeline-l1] L1 failed: ${err instanceof Error ? err.stack ?? err.message : String(err)}`); + // ── error_degradation metric ── + if (instanceId) { + report("error_degradation", { + module: "l1-extraction", + action: "extractL1Memories", + errorType: "exception", + errorMessage: err instanceof Error ? err.message : String(err), + degradedTo: null, + impact: "blocking", + }); + } + throw err; // rethrow so pipeline-manager can retry + } + }); + + // Persister: saves pipeline session states to checkpoint + scheduler.setPersister(async (states) => { + const checkpoint = new CheckpointManager(pluginDataDir, api.logger); + await checkpoint.mergePipelineStates(states); + }); + + // L2 runner: read L1 records (incremental) → SceneExtractor + scheduler.setL2Runner(async (sessionKey: string, cursor?: string) => { + try { + return await runLocalL2Extraction(api, cfg, pluginDataDir, sessionKey, vectorStore, cursor, instanceId); + } catch (err) { + api.logger.error(`${TAG} [pipeline-l2] L2 failed: ${err instanceof Error ? err.stack ?? err.message : String(err)}`); + throw err; // rethrow so pipeline-manager can handle retry/fallback (consistent with L1 runner) + } + }); + + // L3 runner: persona trigger + generation + scheduler.setL3Runner(async () => { + try { + const trigger = new PersonaTrigger({ + dataDir: pluginDataDir, + interval: cfg.persona.triggerEveryN, + logger: api.logger, + }); + + const { should, reason } = await trigger.shouldGenerate(); + if (!should) { + api.logger.debug?.(`${TAG} [pipeline-l3] Persona generation not needed`); + return; + } + + if (!api.config) { + api.logger.warn(`${TAG} [pipeline-l3] No OpenClaw config, skipping persona generation`); + return; + } + + api.logger.info(`${TAG} [pipeline-l3] Starting persona generation: ${reason}`); + const generator = new PersonaGenerator({ + dataDir: pluginDataDir, + config: api.config, + model: cfg.persona.model, + backupCount: cfg.persona.backupCount, + logger: api.logger, + instanceId, + }); + const genResult = await generator.generate(reason); + api.logger.info(`${TAG} [pipeline-l3] Persona generation ${genResult ? "succeeded" : "skipped (no changes)"}`); + } catch (err) { + api.logger.error(`${TAG} [pipeline-l3] Failed: ${err instanceof Error ? err.stack ?? err.message : String(err)}`); + } + }); + + // Capture vectorStore reference for cleanup + const vectorStoreRef = vectorStore; + + // Register a SINGLE gateway_stop hook for ordered shutdown. + // Order: memoryCleaner → scheduler → vectorStore → embeddingService + // (memoryCleaner may use VectorStore during cleanup, so it must stop first) + // + // The entire hook is wrapped with a 3 s timeout to guarantee we never + // block the gateway shutdown path — even if a pipeline flush or DB + // close hangs. Each step is individually timed for observability. + api.on("gateway_stop", async () => { + const GATEWAY_STOP_TIMEOUT_MS = 3_000; + const hookStartMs = Date.now(); + + const doCleanup = async (): Promise => { + // 1. Stop the memory cleaner + if (memoryCleaner) { + try { + memoryCleaner.destroy(); + if (sharedMemoryCleaner === memoryCleaner) { + sharedMemoryCleaner = undefined; + } + } catch (error) { + api.logger.error(`${TAG} [gateway_stop] memoryCleaner error: ${error instanceof Error ? error.message : String(error)}`); + } + } + + // 2. Destroy scheduler (potentially heavy — flushes pending L1/L2/L3) + if (scheduler && schedulerStarted) { + const t = Date.now(); + await scheduler.destroy(); + api.logger.info(`${TAG} [gateway_stop] Scheduler destroyed (${Date.now() - t}ms)`); + } + + // 3. Close VectorStore + if (vectorStoreRef) { + vectorStoreRef.close(); + } + + // 4. Release embedding service resources + if (embeddingService?.close) { + try { + await embeddingService.close(); + } catch (err) { + api.logger.warn(`${TAG} [gateway_stop] EmbeddingService close error: ${err instanceof Error ? err.message : String(err)}`); + } + } + }; + + // Race cleanup against a hard timeout so we never block gateway exit. + let timeoutId: ReturnType | undefined; + try { + await Promise.race([ + doCleanup(), + new Promise((_, reject) => { + timeoutId = setTimeout( + () => reject(new Error("timeout")), + GATEWAY_STOP_TIMEOUT_MS, + ); + }), + ]); + } catch (err) { + api.logger.warn( + `${TAG} [gateway_stop] Aborted (${Date.now() - hookStartMs}ms): ${err instanceof Error ? err.message : String(err)}. ` + + `Pending work will recover on next startup.`, + ); + } finally { + if (timeoutId !== undefined) clearTimeout(timeoutId); + } + + api.logger.info(`${TAG} [gateway_stop] Cleanup finished, all resources released (${Date.now() - hookStartMs}ms)`); + }); + } + + api.logger.info(`${TAG} Registering agent_end hook (auto-capture)`); + api.on("agent_end", async (event, ctx) => { + const startMs = Date.now(); + api.logger.debug?.(`${TAG} [agent_end] Hook triggered`); + + const e = event as Record; + if (!e.success) { + api.logger.info(`${TAG} [agent_end] Agent did not succeed, skipping capture`); + return; + } + + const sessionKey = ctx.sessionKey; + const sessionId = ctx.sessionId; + + if (sessionFilter.shouldSkipCtx(ctx)) { + api.logger.debug?.(`${TAG} [agent_end] Skipping filtered session`); + return; + } + + const messages = (e.messages as unknown[]) ?? []; + const resolvedSessionKey = resolveSessionKey(sessionKey); + if (!resolvedSessionKey) { + return; + } + + // Estimate LLM reasoning time: recallEnd → agentEnd start + const recallEndTs = pendingRecallEndTimestamps.get(resolvedSessionKey); + if (recallEndTs) { + const llmEstimatedMs = startMs - recallEndTs; + api.logger.info( + `${TAG} ⏱ Turn timing: recallEnd→agentEnd=${llmEstimatedMs}ms ` + + `(≈ LLM reasoning + prompt build + tool calls)`, + ); + pendingRecallEndTimestamps.delete(resolvedSessionKey); + } + + // Retrieve cached original prompt (don't delete — retry may trigger multiple agent_end; + // stale entries are swept by TTL in before_prompt_build) + const cachedPrompt = sessionKey ? pendingOriginalPrompts.get(sessionKey) : undefined; + const originalUserText = cachedPrompt?.text; + const originalUserMessageCount = cachedPrompt?.messageCount; + + try { + // Lazy-start the scheduler on first real conversation (Solution C). + // This is a no-op after the first call. + await ensureSchedulerStarted(); + + const captureResult = await performAutoCapture({ + messages, + sessionKey: resolvedSessionKey, + sessionId: sessionId || undefined, + cfg, + pluginDataDir, + logger: api.logger, + scheduler, + originalUserText, + originalUserMessageCount, + pluginStartTimestamp, + vectorStore: sharedVectorStore, + embeddingService: sharedEmbeddingService, + }); + const captureMs = Date.now() - startMs; + api.logger.info( + `${TAG} [agent_end] Auto-capture complete (${captureMs}ms), ` + + `l0Recorded=${captureResult.l0RecordedCount}, ` + + `schedulerNotified=${captureResult.schedulerNotified}`, + ); + + // ── agent_turn metric: one-line trace of the full turn ── + // Retrieve and delete recall cache (delete-after-use to prevent leak) + const cachedRecall = sessionKey ? pendingRecallCache.get(sessionKey) : undefined; + if (sessionKey) pendingRecallCache.delete(sessionKey); + + if (instanceId) { + report("agent_turn", { + sessionKey: resolvedSessionKey, + // User input + userPrompt: originalUserText ?? null, + // Recall results (from before_prompt_build cache) + recalledL1Memories: cachedRecall?.l1Memories ?? [], + recalledL1Count: cachedRecall?.l1Memories?.length ?? 0, + recalledL3Persona: cachedRecall?.l3Persona ?? null, + recallStrategy: cachedRecall?.strategy ?? null, + recallDurationMs: cachedRecall?.durationMs ?? 0, + // L0 write-to-disk results + l0CapturedMessages: captureResult.filteredMessages.map((m) => ({ + role: m.role, + content: m.content, + ts: m.timestamp, + })), + l0CapturedCount: captureResult.l0RecordedCount, + l0VectorsWritten: captureResult.l0VectorsWritten, + // Timing + captureDurationMs: captureMs, + totalDurationMs: Date.now() - startMs, + }); + } + } catch (err) { + const elapsedMs = Date.now() - startMs; + api.logger.error(`${TAG} [agent_end] Auto-capture failed after ${elapsedMs}ms: ${err instanceof Error ? err.stack ?? err.message : String(err)}`); + // ── error_degradation metric ── + if (instanceId) { + report("error_degradation", { + module: "auto-capture", + action: "performAutoCapture", + errorType: "exception", + errorMessage: err instanceof Error ? err.message : String(err), + degradedTo: "no_capture", + impact: "non-blocking", + }); + } + } + }); + } else { + api.logger.info(`${TAG} Auto-capture disabled`); + } + + // memoryCleaner gateway_stop is handled in the unified handler above (inside extraction.enabled block). + // For the case where capture is enabled but extraction is disabled, register cleanup separately. + if (memoryCleaner && !cfg.extraction.enabled) { + api.on("gateway_stop", async () => { + const startMs = Date.now(); + try { + memoryCleaner?.destroy(); + if (sharedMemoryCleaner === memoryCleaner) { + sharedMemoryCleaner = undefined; + } + api.logger.info(`${TAG} [gateway_stop] Memory cleaner destroyed (${Date.now() - startMs}ms)`); + } catch (error) { + api.logger.error(`${TAG} [gateway_stop] Error during memory cleaner destruction (${Date.now() - startMs}ms): ${error instanceof Error ? error.message : String(error)}`); + } + }); + } + + api.logger.info( + `${TAG} Plugin registration complete (v3). ` + + `startTimestamp=${pluginStartTimestamp} (${new Date(pluginStartTimestamp).toISOString()})`, + ); + + // Resolve persistent instance ID for metric reporting (used by agent_turn, error_degradation, etc.) + getOrCreateInstanceId(pluginDataDir).then((id) => { + instanceId = id; + initReporter({ enabled: cfg.report.enabled, type: cfg.report.type, logger: api.logger, instanceId: id, pluginVersion }); + }).catch((err) => { + api.logger.warn(`${TAG} Failed to initialize instanceId for metrics: ${err instanceof Error ? err.message : String(err)}`); + }); +} + +// ============================ +// L2 extraction implementations +// ============================ + +/** + * Local L2 extraction: read L1 records → SceneExtractor. + * + * Uses **incremental** reads when VectorStore is available: + * 1. Receive the pipeline cursor (`last_extraction_updated_time`) from pipeline-manager + * 2. Query only L1 records updated AFTER that cursor via `queryMemoryRecords` + * 3. Return the latest `updatedAt` from the batch so pipeline-manager can advance the cursor + * + * Falls back to JSONL read (with client-side time filtering) when VectorStore is unavailable. + */ +async function runLocalL2Extraction( + api: OpenClawPluginApi, + cfg: MemoryTdaiConfig, + pluginDataDir: string, + sessionKey: string, + vectorStore?: VectorStore, + updatedAfter?: string, + instanceId?: string, +): Promise<{ latestCursor?: string } | void> { + api.logger.debug?.( + `${TAG} [L2-local] session=${sessionKey}, updatedAfter=${updatedAfter ?? "(full)"}`, + ); + + let records: Array<{ content: string; created_at: string; id: string; updatedAt: string }>; + + // Prefer incremental SQLite query when VectorStore is available + if (vectorStore && !vectorStore.isDegraded()) { + const { queryMemoryRecords } = await import("./src/record/l1-reader.js"); + const memRecords = queryMemoryRecords(vectorStore, { + sessionKey, + updatedAfter, + }, api.logger); + + if (memRecords.length === 0) { + api.logger.debug?.( + `${TAG} [L2-local] No new L1 records since cursor (session=${sessionKey}, updatedAfter=${updatedAfter ?? "(full)"}), skipping scene extraction`, + ); + return; + } + + api.logger.debug?.( + `${TAG} [L2-local] Incremental query returned ${memRecords.length} record(s) (session=${sessionKey})`, + ); + + records = memRecords.map((r) => ({ + content: r.content, + created_at: r.createdAt, + id: r.id, + updatedAt: r.updatedAt, + })); + } else { + // Fallback: read JSONL files with client-side time filtering + api.logger.debug?.(`${TAG} [L2-local] VectorStore unavailable, falling back to JSONL read`); + const { readAllMemoryRecords } = await import("./src/record/l1-reader.js"); + let allRecords = await readAllMemoryRecords(pluginDataDir, api.logger); + + // Apply updatedAfter filter on JSONL records (same semantics as SQLite path) + if (updatedAfter) { + const beforeCount = allRecords.length; + allRecords = allRecords.filter((r) => { + const t = r.updatedAt || r.createdAt || ""; + return t > updatedAfter; + }); + api.logger.debug?.( + `${TAG} [L2-local] JSONL time filter: ${beforeCount} → ${allRecords.length} record(s) (updatedAfter=${updatedAfter})`, + ); + } + + if (allRecords.length === 0) { + api.logger.debug?.(`${TAG} [L2-local] No new L1 records found (JSONL fallback), skipping scene extraction`); + return; + } + + records = allRecords.map((r) => ({ + content: r.content, + created_at: r.createdAt, + id: r.id, + updatedAt: r.updatedAt, + })); + } + + const extractor = new SceneExtractor({ + dataDir: pluginDataDir, + config: api.config!, + model: cfg.persona.model, + maxScenes: cfg.persona.maxScenes, + sceneBackupCount: cfg.persona.sceneBackupCount, + logger: api.logger, + instanceId, + }); + + const memories = records.map((r) => ({ + content: r.content, + created_at: r.created_at, + id: r.id, + })); + + // ── Checkpoint guard ────────────────────────────────────────────── + // Snapshot critical counters BEFORE the LLM agent runs. + // The LLM operates on checkpoint via raw file tools (write_to_file / + // replace_in_file) which bypass CheckpointManager's file lock. + // If the LLM accidentally overwrites the entire checkpoint (e.g. via + // write_to_file), system-managed counters like scenes_processed and + // memories_since_last_persona can be reset to stale values. + // After extraction we detect and repair such corruption. + const preCheckpoint = new CheckpointManager(pluginDataDir, api.logger); + const preState = await preCheckpoint.read(); + const preScenesProcessed = preState.scenes_processed; + const preMemoriesSince = preState.memories_since_last_persona; + const preTotalProcessed = preState.total_processed; + + const extractResult = await extractor.extract(memories); + if (extractResult.success && extractResult.memoriesProcessed > 0) { + const checkpoint = new CheckpointManager(pluginDataDir, api.logger); + + // Detect and repair LLM-caused checkpoint corruption. + // If the LLM wrote the entire checkpoint file (instead of using + // replace_in_file on specific fields), system-managed counters may + // have been overwritten with stale/zero values. + const postState = await checkpoint.read(); + if ( + postState.scenes_processed < preScenesProcessed || + postState.total_processed < preTotalProcessed + ) { + api.logger.warn( + `${TAG} [L2-local] ⚠️ Checkpoint corruption detected! ` + + `scenes_processed: ${preScenesProcessed} → ${postState.scenes_processed}, ` + + `total_processed: ${preTotalProcessed} → ${postState.total_processed}, ` + + `memories_since: ${preMemoriesSince} → ${postState.memories_since_last_persona}. ` + + `Repairing...`, + ); + await checkpoint.write({ + ...postState, + scenes_processed: Math.max(postState.scenes_processed, preScenesProcessed), + total_processed: Math.max(postState.total_processed, preTotalProcessed), + memories_since_last_persona: Math.max(postState.memories_since_last_persona, preMemoriesSince), + }); + api.logger.info(`${TAG} [L2-local] Checkpoint repaired`); + } + + await checkpoint.incrementScenesProcessed(); + + // Return the max updatedAt from this batch as the new cursor + const latestCursor = records.reduce((latest, r) => { + return r.updatedAt > latest ? r.updatedAt : latest; + }, ""); + + api.logger.debug?.( + `${TAG} [L2-local] Extraction complete: processed=${extractResult.memoriesProcessed}, latestCursor=${latestCursor}`, + ); + + return { latestCursor: latestCursor || undefined }; + } +} + +// ============================ +// Helpers +// ============================ diff --git a/openclaw.plugin.json b/openclaw.plugin.json new file mode 100644 index 0000000..200f714 --- /dev/null +++ b/openclaw.plugin.json @@ -0,0 +1,90 @@ +{ + "id": "memory-tencentdb", + "name": "Memory (TencentDB)", + "description": "Four-layer memory system — auto-captures, structures, and profiles conversational knowledge", + "configSchema": { + "type": "object", + "additionalProperties": true, + "properties": { + "capture": { + "type": "object", + "description": "对话自动捕获设置 (L0)", + "properties": { + "enabled": { "type": "boolean", "default": true, "description": "是否启用自动对话捕获" }, + "excludeAgents": { "type": "array", "items": { "type": "string" }, "default": [], "description": "排除的 Agent glob 模式列表（如 bench-judge-*），匹配的 agent 不会被捕获、召回或参与 pipeline 调度" }, + "l0l1RetentionDays": { "type": "number", "default": 0, "description": "L0/L1 本地文件保留天数（单位天）。0=不清理；非0时默认必须>=3。若为1或2，需显式开启 allowAggressiveCleanup" }, + "allowAggressiveCleanup": { "type": "boolean", "default": false, "description": "是否允许高风险清理配置（l0l1RetentionDays=1或2）" }, + "cleanTime": { "type": "string", "default": "03:00", "description": "每日清理执行时间（HH:mm）" } + } + }, + "extraction": { + "type": "object", + "description": "记忆提取设置 (L1)", + "properties": { + "enabled": { "type": "boolean", "default": true, "description": "是否启用后台提取" }, + "enableDedup": { "type": "boolean", "default": true, "description": "L1 智能去重（基于向量相似度或关键词进行冲突检测）" }, + "maxMemoriesPerSession": { "type": "number", "default": 20, "description": "单次 L1 提取每 session 最大记忆条数" }, + "model": { "type": "string", "description": "提取使用模型 (格式: provider/model)，未填写时使用openclaw默认模型" } + } + }, + "persona": { + "type": "object", + "description": "场景归纳与用户画像设置 (L2/L3)", + "properties": { + "triggerEveryN": { "type": "number", "default": 50, "description": "每 N 条新记忆触发画像生成" }, + "maxScenes": { "type": "number", "default": 20, "description": "最大场景数" }, + "backupCount": { "type": "number", "default": 3, "description": "画像备份保留数量" }, + "sceneBackupCount": { "type": "number", "default": 10, "description": "场景块备份保留数量" }, + "model": { "type": "string", "description": "画像生成模型 (格式: provider/model)，未填写时使用openclaw默认模型" } + } + }, + "pipeline": { + "type": "object", + "description": "L1→L2→L3 管线调度设置", + "properties": { + "everyNConversations": { "type": "number", "default": 5, "description": "每 N 轮对话触发 L1 批处理" }, + "enableWarmup": { "type": "boolean", "default": true, "description": "Warm-up 模式：新 session 从 1 轮触发开始，每次 L1 后翻倍（1→2→4→...→everyN），加速早期记忆提取" }, + "l1IdleTimeoutSeconds": { "type": "number", "default": 60, "description": "L1 空闲超时（秒）：用户停止对话后多久触发 L1 批处理" }, + "l2DelayAfterL1Seconds": { "type": "number", "default": 90, "description": "L1 完成后延迟多久触发 L2（秒）" }, + "l2MinIntervalSeconds": { "type": "number", "default": 300, "description": "同一 session 两次 L2 抽取的最小间隔（秒）" }, + "l2MaxIntervalSeconds": { "type": "number", "default": 1800, "description": "同一活跃 session 的 L2 最大轮询间隔（秒）" }, + "sessionActiveWindowHours": { "type": "number", "default": 24, "description": "session 活跃窗口（小时），超过此时间不活跃的 session 停止 L2 轮询" } + } + }, + "recall": { + "type": "object", + "description": "记忆召回设置", + "properties": { + "enabled": { "type": "boolean", "default": true, "description": "是否启用自动召回" }, + "maxResults": { "type": "number", "default": 5, "description": "召回最大结果数" }, + "scoreThreshold": { "type": "number", "default": 0.3, "description": "最低分数阈值" }, + "strategy": { "type": "string", "enum": ["embedding", "keyword", "hybrid"], "default": "hybrid", "description": "搜索策略：keyword(关键词)、embedding(向量)、hybrid(混合RRF融合，推荐)" }, + "timeoutMs": { "type": "number", "default": 5000, "description": "召回整体超时（毫秒），超时后跳过记忆注入并打印警告日志" } + } + }, + "embedding": { + "type": "object", + "description": "向量搜索 (Embedding) 配置", + "properties": { + "enabled": { "type": "boolean", "default": true, "description": "是否启用向量搜索" }, + "provider": { "type": "string", "default": "none", "description": "Embedding 服务提供者：填写兼容 OpenAI API 的远端服务名称（如 openai、deepseek 等）；不填或填 none 则禁用向量搜索" }, + "proxyUrl": { "type": "string", "description": "本地代理地址（仅 provider=qclaw 时必填）。配置后 embedding 请求将通过该代理转发，原始 baseUrl 作为 Remote-URL 头传递" }, + "baseUrl": { "type": "string", "description": "API Base URL（必填）：填写对应 provider 的 API 地址" }, + "apiKey": { "type": "string", "description": "API Key（必填）" }, + "model": { "type": "string", "description": "模型名称（必填）" }, + "dimensions": { "type": "number", "description": "向量维度（必填，需与所选模型匹配）" }, + "conflictRecallTopK": { "type": "number", "default": 5, "description": "冲突检测时召回 Top-K 数" }, + "maxInputChars": { "type": "number", "default": 5000, "description": "Embedding 输入文本最大字符数，超出时截断并打印警告日志（默认 5000，适合大多数模型的 token 上限）" } + } + }, + "report": { + "type": "object", + "description": "指标上报设置", + "properties": { + "enabled": { "type": "boolean", "default": false, "description": "是否启用指标上报（通过 Gateway 日志输出结构化 METRIC JSON）" }, + "type": { "type": "string", "default": "local", "description": "上报方式：local 表示通过 logger 输出结构化 JSON 日志" } + } + } + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..ad39271 --- /dev/null +++ b/package.json @@ -0,0 +1,61 @@ +{ + "name": "@tencentdb-agent-memory/memory-tencentdb", + "version": "0.1.1", + "description": "Four-layer local memory system plugin for OpenClaw — auto-captures, structures, and profiles conversational knowledge using local LLM + SQLite vector search (L0→L1→L2→L3 pipeline)", + "type": "module", + "main": "index.ts", + "exports": { + ".": "./index.ts" + }, + "files": [ + "index.ts", + "src/", + "openclaw.plugin.json", + "README.md", + "CHANGELOG.md", + "LICENSE", + "!src/**/*.test.ts", + "!src/**/*.spec.ts", + "!src/**/__tests__/" + ], + "keywords": [ + "openclaw", + "openclaw-plugin", + "memory", + "ai-memory", + "long-term-memory", + "vector-search", + "sqlite-vec", + "llm", + "conversation", + "persona", + "scene-extraction", + "embedding" + ], + "author": "TencentDB Agent Memory Team", + "license": "MIT", + "engines": { + "node": ">=22.16.0" + }, + "dependencies": { + "@node-rs/jieba": "^2.0.1", + "sqlite-vec": "0.1.7-alpha.2" + }, + "peerDependencies": { + "openclaw": ">=2026.3.7", + "node-llama-cpp": "^3.16.2" + }, + "peerDependenciesMeta": { + "openclaw": { + "optional": true + }, + "node-llama-cpp": { + "optional": true + } + }, + "openclaw": { + "extensions": [ + "./index.ts" + ] + } +} diff --git a/src/config.ts b/src/config.ts new file mode 100644 index 0000000..532abf2 --- /dev/null +++ b/src/config.ts @@ -0,0 +1,419 @@ +/** + * Plugin configuration types and parser (v3). + * + * Config is organized into flat functional groups: + * capture, extraction, persona, pipeline, recall, embedding + * + * Minimal config (zero config): {} — all fields have sensible defaults. + */ + +// ============================ +// Type definitions +// ============================ + +/** Capture settings — controls L0 conversation recording. */ +export interface CaptureConfig { + /** Enable auto-capture (default: true) */ + enabled: boolean; + /** Glob patterns to exclude agents (e.g. "bench-judge-*"); matched agents are fully ignored */ + excludeAgents: string[]; + /** + * L0/L1 local file retention days used as TTL switch. + * 0 means cleanup disabled.(default: 0) + */ + l0l1RetentionDays: number; + + /** + * Allow dangerous low retention (1 or 2 days). + * Default false: when disabled, non-zero retention must be >= 3. + */ + allowAggressiveCleanup: boolean; +} + +/** Extraction settings (L1) — controls memory extraction from conversations. */ +export interface ExtractionConfig { + /** Enable background extraction (default: true) */ + enabled: boolean; + /** Enable L1 smart dedup (default: true) */ + enableDedup: boolean; + /** Max memories per session (default: 20) */ + maxMemoriesPerSession: number; + /** LLM model for extraction, format: "provider/model" (falls back to OpenClaw default model when omitted) */ + model?: string; +} + +/** Persona (L2/L3) settings — controls scene extraction (L2) and user profile generation (L3). */ +export interface PersonaConfig { + /** Trigger persona generation every N new memories (default: 50) */ + triggerEveryN: number; + /** Max scene blocks (default: 20) */ + maxScenes: number; + /** Persona backup count (default: 3) */ + backupCount: number; + /** Scene blocks backup count (default: 10) */ + sceneBackupCount: number; + /** LLM model for persona generation, format: "provider/model" (falls back to OpenClaw default model when omitted) */ + model?: string; +} + +/** Pipeline trigger settings (L1→L2→L3 scheduling). */ +export interface PipelineTriggerConfig { + /** Trigger L1 after every N conversation rounds (default: 5) */ + everyNConversations: number; + /** Enable warm-up: start threshold at 1, double after each L1 (1→2→4→...→everyN) (default: true) */ + enableWarmup: boolean; + /** L1 idle timeout: trigger L1 after this many seconds of inactivity (default: 60) */ + l1IdleTimeoutSeconds: number; + /** L2 delay after L1: wait this many seconds after L1 completes before triggering L2 (default: 90) */ + l2DelayAfterL1Seconds: number; + /** L2 min interval: minimum seconds between L2 runs per session (default: 300 = 5 min) */ + l2MinIntervalSeconds: number; + /** L2 max interval: even without new conversations, trigger L2 at most this often per session (default: 1800 = 30 min) */ + l2MaxIntervalSeconds: number; + /** Sessions inactive longer than this (hours) stop L2 polling (default: 24) */ + sessionActiveWindowHours: number; +} + +/** Recall settings — controls memory retrieval for context injection. */ +export interface RecallConfig { + /** Enable auto-recall (default: true) */ + enabled: boolean; + /** Max results to return (default: 5) */ + maxResults: number; + /** Minimum score threshold (default: 0.3) */ + scoreThreshold: number; + /** Search strategy (default: "hybrid") */ + strategy: "embedding" | "keyword" | "hybrid"; + /** Overall recall timeout in milliseconds (default: 5000). When exceeded, recall is skipped with a warning. */ + timeoutMs: number; +} + +/** Embedding service configuration for vector search. */ +export interface EmbeddingConfig { + /** User-facing default is true in schema, but provider="none" still disables embedding effectively. */ + enabled: boolean; + /** Embedding provider: default "none" disables vector search; other values (e.g. "openai", "deepseek") are treated as OpenAI-compatible remote providers. */ + provider: string; + /** API Base URL (required for remote provider). */ + baseUrl: string; + /** API Key (required for remote provider). */ + apiKey: string; + /** Model name (required for remote provider). */ + model: string; + /** Vector dimensions (required for remote provider, must match model). */ + dimensions: number; + /** Top-K candidates to recall during conflict detection (default: 5) */ + conflictRecallTopK: number; + /** Proxy URL for qclaw provider — when provider="qclaw", requests are forwarded through this local proxy */ + proxyUrl?: string; + /** Max input text length in characters before truncation (default: 5000). Texts exceeding this limit are truncated with a warning. */ + maxInputChars: number; + /** Timeout per embedding API call in milliseconds (default: 10000). */ + timeoutMs: number; + /** Internal-only local model cache directory, not exposed in plugin schema. */ + modelCacheDir?: string; + /** If set, contains an error message about invalid remote config (embedding is disabled) */ + configError?: string; +} + +/** Daily cleaner settings for local JSONL data (L0/L1). */ +export interface MemoryCleanupConfig { + /** TTL switch from capture.l0l1RetentionDays. Undefined means disabled. */ + retentionDays?: number; + + /** Whether cleanup is enabled. True only when retentionDays is a valid positive number. */ + enabled: boolean; + /** Daily execution time in HH:mm format (default: 03:00). */ + cleanTime: string; +} + +/** Report settings — controls metric/event reporting. */ +export interface ReportConfig { + /** Enable reporting (default: true) */ + enabled: boolean; + /** Reporter type: "local" logs structured JSON via logger (default: "local") */ + type: string; +} + +/** Fully resolved plugin configuration (v3). */ +export interface MemoryTdaiConfig { + capture: CaptureConfig; + extraction: ExtractionConfig; + persona: PersonaConfig; + pipeline: PipelineTriggerConfig; + recall: RecallConfig; + embedding: EmbeddingConfig; + memoryCleanup: MemoryCleanupConfig; + report: ReportConfig; +} + +// ============================ +// Parser +// ============================ + +/** + * Parse plugin config from raw user input. + * All fields have sensible defaults — minimal config is just {}. + */ +export function parseConfig(raw: Record | undefined): MemoryTdaiConfig { + const c = raw ?? {}; + + // --- Capture (L0) --- + const captureGroup = obj(c, "capture"); + + // --- Retention days validation (from capture.l0l1RetentionDays) --- + const rawRetentionDays = num(captureGroup, "l0l1RetentionDays") ?? 0; + const allowAggressiveCleanup = bool(captureGroup, "allowAggressiveCleanup") ?? false; + + let retentionDays: number | undefined; + if (rawRetentionDays <= 0) { + retentionDays = undefined; + } else if (rawRetentionDays >= 3) { + retentionDays = rawRetentionDays; + } else if (allowAggressiveCleanup) { + retentionDays = rawRetentionDays; + } else { + retentionDays = undefined; + } + + // --- Extraction (L1) --- + const extractionGroup = obj(c, "extraction"); + + // --- Persona (L2/L3) --- + const personaGroup = obj(c, "persona"); + + // --- Pipeline --- + const pipelineGroup = obj(c, "pipeline"); + + // --- Recall --- + const recallGroup = obj(c, "recall"); + + // --- Embedding --- + const embeddingGroup = obj(c, "embedding"); + let embeddingConfigError: string | undefined; + + // Embedding config: determine provider based on user input and apiKey availability + const embeddingApiKey = str(embeddingGroup, "apiKey") ?? ""; + const embeddingBaseUrl = str(embeddingGroup, "baseUrl") ?? ""; + const embeddingProviderRaw = str(embeddingGroup, "provider") ?? "none"; + const embeddingModelRaw = str(embeddingGroup, "model") ?? ""; + const embeddingDimensionsRaw = num(embeddingGroup, "dimensions"); + const embeddingProxyUrl = str(embeddingGroup, "proxyUrl"); + + // provider="none" → embedding disabled (default for zero-config users) + // provider="local" → no longer exposed to users; treated as disabled at entry level + // provider="qclaw" → requires proxyUrl for local proxy forwarding + // Any other value → remote mode (requires apiKey, baseUrl, model, dimensions) + let embeddingProvider: string; + let embeddingEnabled = bool(embeddingGroup, "enabled") ?? true; + + if (embeddingProviderRaw === "none") { + // Explicitly disabled (default): no embedding, no vector search + embeddingProvider = "none"; + embeddingEnabled = false; + } else if (embeddingProviderRaw === "local") { + // Local embedding is not exposed to users; treat as disabled at entry level. + // Internal LocalEmbeddingService code is preserved but not reachable from config. + embeddingProvider = "none"; + embeddingEnabled = false; + embeddingConfigError = + "Local embedding provider is not available in user config. " + + "Please configure a remote embedding provider (e.g. openai, deepseek). Embedding has been disabled."; + } else if (embeddingProviderRaw === "qclaw") { + // qclaw provider: requires proxyUrl for local proxy forwarding + const missingFields: string[] = []; + if (!embeddingProxyUrl) missingFields.push("proxyUrl"); + if (!embeddingBaseUrl) missingFields.push("baseUrl"); + if (!embeddingApiKey) missingFields.push("apiKey"); + if (!embeddingModelRaw) missingFields.push("model"); + if (embeddingDimensionsRaw == null || embeddingDimensionsRaw <= 0) missingFields.push("dimensions"); + + if (missingFields.length > 0) { + const errorMsg = + `Embedding provider 'qclaw' requires 'proxyUrl', 'baseUrl', 'apiKey', 'model', and 'dimensions' to be set. ` + + `Missing: ${missingFields.join(", ")}. Embedding has been disabled.`; + embeddingConfigError = errorMsg; + embeddingEnabled = false; + embeddingProvider = embeddingProviderRaw; + } else { + embeddingProvider = embeddingProviderRaw; + } + } else { + // Remote mode — validate all required fields + const missingFields: string[] = []; + if (!embeddingApiKey) missingFields.push("apiKey"); + if (!embeddingBaseUrl) missingFields.push("baseUrl"); + if (!embeddingModelRaw) missingFields.push("model"); + if (embeddingDimensionsRaw == null || embeddingDimensionsRaw <= 0) missingFields.push("dimensions"); + + if (missingFields.length > 0) { + // Configuration error: disable embedding and log detailed error + // This does NOT throw — the plugin continues running without vector search + const errorMsg = + `Remote embedding provider '${embeddingProviderRaw}' requires 'apiKey', 'baseUrl', 'model', and 'dimensions' to be set. ` + + `Missing: ${missingFields.join(", ")}. Embedding has been disabled.`; + // We store the error message so the caller (index.ts) can log it + embeddingConfigError = errorMsg; + embeddingEnabled = false; + embeddingProvider = embeddingProviderRaw; // preserve original for error context + } else { + embeddingProvider = embeddingProviderRaw; + } + } + + // When provider="none", dimensions=0 signals VectorStore to skip vec0 table + // creation entirely (deferred until a real embedding provider is configured). + // This avoids creating vec0 tables with a placeholder dimension that would + // mismatch if the user later enables a different-dimensional provider. + const defaultDimensions = + embeddingProvider === "none" ? 0 : + embeddingDimensionsRaw ?? 0; + const defaultModel = embeddingProvider === "none" ? "" : embeddingModelRaw; + + const cleanTime = normalizeCleanTime(str(captureGroup, "cleanTime")) ?? "03:00"; + + const memoryCleanup: MemoryCleanupConfig = { + retentionDays, + enabled: retentionDays != null, + cleanTime, + }; + + return { + capture: { + enabled: bool(captureGroup, "enabled") ?? true, + excludeAgents: strArray(captureGroup, "excludeAgents") ?? [], + l0l1RetentionDays: retentionDays ?? 0, + allowAggressiveCleanup, + }, + extraction: { + enabled: bool(extractionGroup, "enabled") ?? true, + enableDedup: bool(extractionGroup, "enableDedup") ?? true, + maxMemoriesPerSession: num(extractionGroup, "maxMemoriesPerSession") ?? 20, + model: optStr(extractionGroup, "model"), + }, + persona: { + triggerEveryN: num(personaGroup, "triggerEveryN") ?? 50, + maxScenes: num(personaGroup, "maxScenes") ?? 20, + backupCount: num(personaGroup, "backupCount") ?? 3, + sceneBackupCount: num(personaGroup, "sceneBackupCount") ?? 10, + model: optStr(personaGroup, "model"), + }, + pipeline: { + everyNConversations: num(pipelineGroup, "everyNConversations") ?? 5, + enableWarmup: bool(pipelineGroup, "enableWarmup") ?? true, + l1IdleTimeoutSeconds: num(pipelineGroup, "l1IdleTimeoutSeconds") ?? 60, + l2DelayAfterL1Seconds: num(pipelineGroup, "l2DelayAfterL1Seconds") ?? 90, + l2MinIntervalSeconds: num(pipelineGroup, "l2MinIntervalSeconds") ?? 300, + l2MaxIntervalSeconds: num(pipelineGroup, "l2MaxIntervalSeconds") ?? 1800, + sessionActiveWindowHours: num(pipelineGroup, "sessionActiveWindowHours") ?? 24, + }, + recall: { + enabled: bool(recallGroup, "enabled") ?? true, + maxResults: num(recallGroup, "maxResults") ?? 5, + scoreThreshold: num(recallGroup, "scoreThreshold") ?? 0.3, + strategy: validateStrategy(str(recallGroup, "strategy")) ?? "hybrid", + timeoutMs: num(recallGroup, "timeoutMs") ?? 5000, + }, + embedding: { + enabled: embeddingEnabled, + provider: embeddingProvider, + baseUrl: embeddingBaseUrl, + apiKey: embeddingApiKey, + model: str(embeddingGroup, "model") ?? defaultModel, + dimensions: num(embeddingGroup, "dimensions") ?? defaultDimensions, + conflictRecallTopK: num(embeddingGroup, "conflictRecallTopK") ?? 5, + proxyUrl: embeddingProxyUrl, + maxInputChars: num(embeddingGroup, "maxInputChars") ?? 5000, + timeoutMs: num(embeddingGroup, "timeoutMs") ?? 10_000, + modelCacheDir: optStr(embeddingGroup, "modelCacheDir"), + configError: embeddingConfigError, + }, + memoryCleanup, + report: { + enabled: bool(obj(c, "report"), "enabled") ?? false, + type: str(obj(c, "report"), "type") ?? "local", + }, + }; +} + +// ============================ +// Helper functions +// ============================ + +/** Get sub-object by key, or empty object if missing. */ +function obj(c: Record, key: string): Record { + const v = c[key]; + return v && typeof v === "object" && !Array.isArray(v) ? v as Record : {}; +} + +function str(src: Record, key: string): string | undefined { + const v = src[key]; + return typeof v === "string" && v.trim() ? v.trim() : undefined; +} + +function optStr(src: Record, key: string): string | undefined { + const v = src[key]; + return typeof v === "string" ? v : undefined; +} + +function num(src: Record, key: string): number | undefined { + const v = src[key]; + return typeof v === "number" && Number.isFinite(v) ? v : undefined; +} + +function bool(src: Record, key: string): boolean | undefined { + const v = src[key]; + return typeof v === "boolean" ? v : undefined; +} + +function strArray(src: Record, key: string): string[] | undefined { + const v = src[key]; + if (!Array.isArray(v)) return undefined; + return v.filter((item): item is string => typeof item === "string" && item.trim().length > 0); +} + +const VALID_STRATEGIES: RecallConfig["strategy"][] = ["embedding", "keyword", "hybrid"]; + +/** + * Validate recall strategy against whitelist. + * Returns the strategy if valid, undefined otherwise (caller falls back to default). + */ +function validateStrategy(value: string | undefined): RecallConfig["strategy"] | undefined { + if (!value) return undefined; + return VALID_STRATEGIES.includes(value as RecallConfig["strategy"]) + ? (value as RecallConfig["strategy"]) + : undefined; +} + +/** + * Normalize a cleanup time string. + * + * The input must follow "HH:MM" or "H:MM" format (24-hour clock). + * If the time is valid, it returns the normalized format "HH:MM" + * with leading zeros added when necessary. + * If the format is invalid or the time is out of range + * (hour: 0–23, minute: 0–59), it returns undefined. + * + * Examples: + * normalizeCleanTime("3:05") -> "03:05" + * normalizeCleanTime("03:05") -> "03:05" + * normalizeCleanTime("23:59") -> "23:59" + * + * normalizeCleanTime("24:00") -> undefined // hour out of range + * normalizeCleanTime("12:60") -> undefined // minute out of range + * normalizeCleanTime("3:5") -> undefined // minute must have two digits + * normalizeCleanTime("abc") -> undefined // invalid format + */ +function normalizeCleanTime(input: string | undefined): string | undefined { + if (!input) return undefined; + const trimmed = input.trim(); + const m = /^(\d{1,2}):(\d{2})$/.exec(trimmed); + if (!m) return undefined; + + const hh = Number(m[1]); + const mm = Number(m[2]); + if (!Number.isInteger(hh) || !Number.isInteger(mm)) return undefined; + if (hh < 0 || hh > 23 || mm < 0 || mm > 59) return undefined; + + return `${String(hh).padStart(2, "0")}:${String(mm).padStart(2, "0")}`; +} diff --git a/src/conversation/l0-recorder.ts b/src/conversation/l0-recorder.ts new file mode 100644 index 0000000..f0dd7dd --- /dev/null +++ b/src/conversation/l0-recorder.ts @@ -0,0 +1,579 @@ +/** + * L0 Conversation Recorder: records raw conversation messages to local JSONL files. + * + * Triggered from agent_end hook. Receives the conversation messages directly from + * the hook context (no file I/O needed), sanitizes them, filters out noise, and + * writes to ~/.openclaw/memory-tdai/conversations/YYYY-MM-DD.jsonl + * + * Design decisions: + * - Uses JSONL format (**one message per line** — flat, easy to grep/stream) + * - One file per day (all sessions merged into the same daily file) + * - sessionKey is stored as a field in each JSONL line, not in the filename + * - Independent from system session files — format fully controlled by plugin + * - Messages are sanitized to remove injected tags (prevent feedback loops) + * - Short/long/command messages are filtered out + */ + +import fs from "node:fs/promises"; +import path from "node:path"; +import crypto from "node:crypto"; +import { sanitizeText, stripCodeBlocks, shouldCaptureL0 } from "../utils/sanitize.js"; + +// ============================ +// Types +// ============================ + +export interface ConversationMessage { + /** Unique message ID (used by L1 prompt for source_message_ids tracking) */ + id: string; + role: "user" | "assistant"; + content: string; + timestamp: number; // epoch ms +} + +/** + * Generate a short unique message ID. + */ +function generateMessageId(): string { + return `msg_${Date.now()}_${crypto.randomBytes(3).toString("hex")}`; +} + +/** + * New flat format: one message per JSONL line. + */ +export interface L0MessageRecord { + sessionKey: string; + sessionId: string; + recordedAt: string; // ISO timestamp + id: string; + role: "user" | "assistant"; + content: string; + timestamp: number; // epoch ms +} + +/** + * A group of conversation messages (used by downstream consumers). + * Each L0ConversationRecord represents one or more messages from the same recording event. + */ +export interface L0ConversationRecord { + sessionKey: string; + sessionId: string; + recordedAt: string; // ISO timestamp + messageCount: number; + messages: ConversationMessage[]; +} + +interface Logger { + debug?: (message: string) => void; + info: (message: string) => void; + warn: (message: string) => void; + error: (message: string) => void; +} + +const TAG = "[memory-tdai][l0]"; + +// ============================ +// Core function +// ============================ + +/** + * Record a conversation round to the L0 JSONL file. + * + * Only records **incremental** messages (new since the last capture). + * Uses `afterTimestamp` as the primary filter to skip already-captured history. + * + * @param sessionKey - The session key for this conversation + * @param rawMessages - Raw messages from the agent_end hook context (full session history) + * @param baseDir - Base data directory (~/.openclaw/memory-tdai/) + * @param logger - Optional logger + * @param originalUserText - Clean original user prompt (pre-prependContext) + * @param afterTimestamp - Epoch ms cursor: only messages with timestamp > this are new. + * Pass 0 or omit for the first capture of a session. + * @returns Filtered messages (for L1 to use directly), or empty array if nothing worth recording + */ +export async function recordConversation(params: { + sessionKey: string; + sessionId?: string; + rawMessages: unknown[]; + baseDir: string; + logger?: Logger; + /** Clean original user prompt (pre-prependContext) */ + originalUserText?: string; + /** Epoch ms cursor: only process messages with timestamp strictly greater than this. */ + afterTimestamp?: number; + /** + * Number of messages in the session at before_prompt_build time. + * Used to locate the exact user message that originalUserText corresponds to: + * rawMessages[originalUserMessageCount] is the user message appended by the framework + * AFTER before_prompt_build, i.e. the one whose content was polluted by prependContext. + */ + originalUserMessageCount?: number; +}): Promise { + const { sessionKey, sessionId, rawMessages, baseDir, logger, originalUserText, afterTimestamp, originalUserMessageCount } = params; + + // Step 1: Position slice + extract user/assistant messages. + // + // Dual protection against duplicate capture: + // Layer 1 (position slice): Use originalUserMessageCount (cached at before_prompt_build) + // to slice rawMessages — only keep messages added AFTER the prompt build, i.e. this + // turn's new messages. This is immune to timestamp drift after gateway restarts. + // Layer 2 (timestamp cursor): The existing afterTimestamp filter below acts as a fallback + // when the position slice is unavailable (cache expired, process restart, etc.). + const usePositionSlice = originalUserMessageCount != null && originalUserMessageCount > 0 + && originalUserMessageCount <= rawMessages.length; + const slicedMessages = usePositionSlice + ? rawMessages.slice(originalUserMessageCount) + : rawMessages; + + const allExtracted = extractUserAssistantMessages(slicedMessages); + + if (usePositionSlice) { + logger?.debug?.( + `${TAG} Position slice: ${rawMessages.length} raw → ${slicedMessages.length} new (sliceStart=${originalUserMessageCount})`, + ); + } + + // Diagnostic: check whether the framework actually provides timestamp on raw messages. + // If all raw timestamps are missing, the timestamp cursor is effectively useless and + // position slice becomes the sole incremental mechanism. + if (slicedMessages.length > 0) { + const firstRaw = slicedMessages[0] as Record | undefined; + const rawTs = firstRaw?.timestamp; + const hasRawTs = typeof rawTs === "number"; + logger?.debug?.( + `${TAG} Raw message[0] timestamp probe: ${hasRawTs ? `present (${rawTs})` : `missing (type=${typeof rawTs}, value=${String(rawTs)})`}`, + ); + } + + logger?.debug?.(`${TAG} Extracted ${allExtracted.length} user/assistant messages from ${slicedMessages.length} total`); + + // Step 1.5: Incremental filter — only keep messages newer than the cursor. + // + // Uses strict greater-than (>) which is safe because: + // - The cursor is set to max(timestamps) of the LAST recorded batch. + // - The next agent turn's messages will have timestamps strictly greater than + // the previous turn (there's at least one LLM API call between turns, which + // takes hundreds of milliseconds minimum — no same-millisecond collision). + // - All messages within a single turn are captured together as one batch, + // so even if multiple messages share the same timestamp, they are either + // all included (new batch) or all excluded (already captured). + // - If a message lacks a timestamp field, extractUserAssistantMessages() + // assigns Date.now() at extraction time, which is always > previous cursor. + const cursor = afterTimestamp ?? 0; + const extracted = cursor > 0 + ? allExtracted.filter((m) => m.timestamp > cursor) + : allExtracted; + + if (extracted.length > 0) { + const first = extracted[0]; + logger?.debug?.( + `${TAG} First captured message: role=${first.role}, ts=${first.timestamp}, ` + + `date=${new Date(first.timestamp).toISOString()}, content=${first.content.slice(0, 80)}${first.content.length > 80 ? "…" : ""}`, + ); + } + + if (cursor > 0) { + logger?.debug?.( + `${TAG} Incremental filter: ${allExtracted.length} total → ${extracted.length} new (cursor=${cursor})`, + ); + + // Safety valve: if timestamp filter passed everything through and position slice + // was not available, this likely indicates timestamp drift after a gateway restart. + if (!usePositionSlice && extracted.length === allExtracted.length && allExtracted.length > 8) { + logger?.warn?.( + `${TAG} ⚠ Safety valve: all ${allExtracted.length} messages passed timestamp filter (cursor=${cursor}) — ` + + `possible timestamp drift after gateway restart. Position slice was not available (no cached messageCount).`, + ); + } + } + + if (extracted.length === 0) { + logger?.debug?.(`${TAG} No new user/assistant messages to record`); + return []; + } + + // Step 2: Replace polluted user messages with cached original prompt. + // + // Background: + // The framework appends the user's message to the session after before_prompt_build, + // then injects prependContext into it. So the user message in rawMessages is polluted. + // We cached the clean prompt (originalUserText) and the message count at + // before_prompt_build time (originalUserMessageCount) to identify which raw message + // is the real user input. + // + // Strategy: + // When position slice is active, the polluted user message is slicedMessages[0]. + // Otherwise, fall back to rawMessages[originalUserMessageCount]. + // In both cases, find the timestamp and match it in `extracted` for replacement. + // If matching fails, skip replacement — sanitizeText() in Step 3 is the safety net. + if (originalUserText) { + // Determine the target raw message that contains the polluted user prompt + const targetRaw: Record | undefined = usePositionSlice + ? slicedMessages[0] as Record | undefined + : (originalUserMessageCount != null && originalUserMessageCount >= 0 && originalUserMessageCount < rawMessages.length) + ? rawMessages[originalUserMessageCount] as Record | undefined + : undefined; + + const targetTs = targetRaw && typeof targetRaw.timestamp === "number" ? targetRaw.timestamp : undefined; + + if (targetTs != null) { + let replaced = false; + for (let i = 0; i < extracted.length; i++) { + if (extracted[i].role === "user" && extracted[i].timestamp === targetTs) { + logger?.debug?.( + `${TAG} Replacing user message at timestamp=${targetTs} with cached original prompt ` + + `(${originalUserText.length} chars, was ${extracted[i].content.length} chars) [positionSlice=${usePositionSlice}]`, + ); + extracted[i] = { ...extracted[i], content: originalUserText }; + replaced = true; + break; + } + } + if (!replaced) { + logger?.warn?.( + `${TAG} Target user message (ts=${targetTs}) not found in extracted batch — ` + + `possibly filtered by cursor. Skipping replacement, will rely on sanitizeText().`, + ); + } + } else if (targetRaw) { + logger?.warn?.( + `${TAG} Target raw message has no valid timestamp — ` + + `skipping replacement, will rely on sanitizeText().`, + ); + } else { + logger?.warn?.( + `${TAG} Have originalUserText but cannot locate target raw message — ` + + `skipping replacement, will rely on sanitizeText().`, + ); + } + } + + // Step 3: Sanitize and filter + const filtered = extracted + .map((m) => { + let content = sanitizeText(m.content); + // Strip fenced code blocks from assistant replies to reduce embedding noise + if (m.role === "assistant") { + content = stripCodeBlocks(content); + } + return { id: m.id, role: m.role, content, timestamp: m.timestamp }; + }) + .filter((m) => shouldCaptureL0(m.content)); + + logger?.debug?.(`${TAG} After sanitize+filter: ${filtered.length} messages (from ${extracted.length})`); + + if (filtered.length === 0) { + logger?.debug?.(`${TAG} All messages filtered out, skipping L0 write`); + return []; + } + + // Step 4: Write to JSONL file — one message per line (flat format) + const now = new Date().toISOString(); + const lines: string[] = []; + for (const msg of filtered) { + const record: L0MessageRecord = { + sessionKey, + sessionId: sessionId || "", + recordedAt: now, + id: msg.id, + role: msg.role, + content: msg.content, + timestamp: msg.timestamp, + }; + lines.push(JSON.stringify(record)); + } + + const shardDate = formatLocalDate(new Date()); + const outDir = path.join(baseDir, "conversations"); + const outPath = path.join(outDir, `${shardDate}.jsonl`); + + try { + await fs.mkdir(outDir, { recursive: true }); + // Append each message as its own JSONL line + await fs.appendFile(outPath, lines.join("\n") + "\n", "utf-8"); + logger?.debug?.(`${TAG} Recorded ${filtered.length} messages to ${outPath}`); + } catch (err) { + logger?.error(`${TAG} Failed to write L0 file: ${err instanceof Error ? err.message : String(err)}`); + // Return filtered messages anyway so L1 can still process them + } + + return filtered; +} + +/** + * Read all L0 conversation records for a session. + * Returns records in chronological order. + * + * File format: `YYYY-MM-DD.jsonl` (daily files, all sessions merged). + * Each line is an L0MessageRecord; filtered by sessionKey at line level. + */ +export async function readConversationRecords( + sessionKey: string, + baseDir: string, + logger?: Logger, +): Promise { + const conversationsDir = path.join(baseDir, "conversations"); + + // Daily file pattern: YYYY-MM-DD.jsonl + const dateFilePattern = /^\d{4}-\d{2}-\d{2}\.jsonl$/; + + let entries: string[]; + try { + const dirEntries = await fs.readdir(conversationsDir, { withFileTypes: true }); + entries = dirEntries + .filter((entry) => entry.isFile()) + .map((entry) => entry.name); + } catch { + // Directory doesn't exist yet — normal for first conversation + return []; + } + + const targetFiles = entries + .filter((name) => dateFilePattern.test(name)) + .sort(); + + if (targetFiles.length === 0) { + return []; + } + + const records: L0ConversationRecord[] = []; + + for (const fileName of targetFiles) { + const filePath = path.join(conversationsDir, fileName); + + let raw: string; + try { + raw = await fs.readFile(filePath, "utf-8"); + } catch { + logger?.warn?.(`${TAG} Failed to read L0 file: ${filePath}`); + continue; + } + + const lines = raw.split("\n").filter((line: string) => line.trim()); + for (let i = 0; i < lines.length; i++) { + const line = lines[i]; + try { + const parsed = JSON.parse(line) as Record; + + // Filter by sessionKey at line level + const lineSessionKey = parsed.sessionKey as string | undefined; + if (lineSessionKey !== sessionKey) continue; + + if (typeof parsed.role === "string" && typeof parsed.content === "string") { + // Flat format: { sessionKey, sessionId, recordedAt, id, role, content, timestamp } + // Wrap into L0ConversationRecord for uniform downstream consumption + const msg: ConversationMessage = { + id: (typeof parsed.id === "string" && parsed.id) ? parsed.id : generateMessageId(), + role: parsed.role as "user" | "assistant", + content: parsed.content as string, + timestamp: typeof parsed.timestamp === "number" ? parsed.timestamp : Date.now(), + }; + records.push({ + sessionKey: (parsed.sessionKey as string) || sessionKey, + sessionId: (parsed.sessionId as string) || "", + recordedAt: (parsed.recordedAt as string) || new Date().toISOString(), + messageCount: 1, + messages: [msg], + }); + } else { + logger?.warn?.(`${TAG} Unrecognized JSONL line format in ${filePath}:${i + 1}`); + } + } catch { + logger?.warn?.(`${TAG} Skipping malformed JSONL line in ${filePath}:${i + 1}`); + } + } + } + + records.sort((a, b) => { + const ta = Date.parse(a.recordedAt); + const tb = Date.parse(b.recordedAt); + const na = Number.isFinite(ta) ? ta : Number.POSITIVE_INFINITY; + const nb = Number.isFinite(tb) ? tb : Number.POSITIVE_INFINITY; + return na - nb; + }); + + return records; +} + +/** + * Read L0 messages across all conversation records for a session, + * optionally filtered by a cursor timestamp (messages after the cursor). + * + * When `limit` is provided, only the **newest** `limit` messages are returned + * (matching the DB path's `ORDER BY timestamp DESC LIMIT ?` behavior). + * Returned messages are always in chronological order (oldest → newest). + * + * NOTE: potential optimization — records are chronologically ordered (append-only JSONL), + * so a reverse scan could skip entire old records. Deferred for now; see Issue 5 in + * docs/05-known-issues.md. + */ +export async function readConversationMessages( + sessionKey: string, + baseDir: string, + afterTimestamp?: number, + logger?: Logger, + limit?: number, +): Promise { + const records = await readConversationRecords(sessionKey, baseDir, logger); + const allMessages: ConversationMessage[] = []; + + for (const record of records) { + for (const msg of record.messages) { + if (afterTimestamp && msg.timestamp <= afterTimestamp) continue; + allMessages.push(msg); + } + } + + // Truncate to newest `limit` messages (keep tail, since array is chronological) + if (limit != null && limit > 0 && allMessages.length > limit) { + logger?.debug?.( + `${TAG} readConversationMessages: truncating ${allMessages.length} → ${limit} (newest)`, + ); + return allMessages.slice(-limit); + } + + return allMessages; +} + +/** + * A group of conversation messages sharing the same sessionId. + */ +export interface SessionIdMessageGroup { + sessionId: string; + messages: ConversationMessage[]; +} + +/** + * Read L0 messages for a session, grouped by sessionId. + * + * Within the same sessionKey, different sessionIds represent different conversation + * instances (e.g. after /reset). L1 extraction should process each group independently + * so that each group's sessionId is correctly associated with its extracted memories. + * + * When `limit` is provided, only the **newest** `limit` messages (across all groups) + * are retained — matching the DB path's `ORDER BY timestamp DESC LIMIT ?` behavior. + * Groups that become empty after truncation are dropped. + * + * Groups are returned in chronological order (by earliest message timestamp). + * Messages within each group are also in chronological order. + */ +export async function readConversationMessagesGroupedBySessionId( + sessionKey: string, + baseDir: string, + afterTimestamp?: number, + logger?: Logger, + limit?: number, +): Promise { + const records = await readConversationRecords(sessionKey, baseDir, logger); + + // Collect all messages with their sessionId, respecting afterTimestamp filter + const allMessages: Array<{ sessionId: string; msg: ConversationMessage }> = []; + + for (const record of records) { + const sid = record.sessionId || ""; + for (const msg of record.messages) { + if (afterTimestamp && msg.timestamp <= afterTimestamp) continue; + allMessages.push({ sessionId: sid, msg }); + } + } + + // Sort by timestamp ASC (chronological) — records are already roughly ordered + // by recordedAt, but messages within may not be perfectly sorted by timestamp. + allMessages.sort((a, b) => a.msg.timestamp - b.msg.timestamp); + + // Truncate to newest `limit` messages (keep tail) + let selected = allMessages; + if (limit != null && limit > 0 && allMessages.length > limit) { + logger?.debug?.( + `${TAG} readConversationMessagesGroupedBySessionId: truncating ${allMessages.length} → ${limit} (newest)`, + ); + selected = allMessages.slice(-limit); + } + + // Re-group by sessionId + const groupMap = new Map(); + for (const { sessionId, msg } of selected) { + let group = groupMap.get(sessionId); + if (!group) { + group = []; + groupMap.set(sessionId, group); + } + group.push(msg); + } + + // Convert to array, sorted by earliest message timestamp in each group + const groups: SessionIdMessageGroup[] = []; + for (const [sessionId, messages] of groupMap) { + if (messages.length > 0) { + groups.push({ sessionId, messages }); + } + } + groups.sort((a, b) => a.messages[0].timestamp - b.messages[0].timestamp); + + return groups; +} + +// ============================ +// Helpers +// ============================ + +/** + * Extract user and assistant messages from raw hook message array. + */ +function extractUserAssistantMessages(messages: unknown[]): ConversationMessage[] { + const result: ConversationMessage[] = []; + + for (const msg of messages) { + if (!msg || typeof msg !== "object") continue; + const m = msg as Record; + const role = m.role as string | undefined; + + if (role !== "user" && role !== "assistant") continue; + + let content: string | undefined; + if (typeof m.content === "string") { + content = m.content; + } else if (Array.isArray(m.content)) { + const textParts: string[] = []; + for (const part of m.content) { + if ( + part && + typeof part === "object" && + (part as Record).type === "text" + ) { + const text = (part as Record).text; + if (typeof text === "string") textParts.push(text); + } + } + content = textParts.join("\n"); + } + + // Strip inline base64 image data URIs that some providers embed in string content. + // These are not useful for memory and would pollute FTS / embedding indexes. + if (content && /data:image\/[a-z+]+;base64,/i.test(content)) { + content = content.replace(/data:image\/[a-z+]+;base64,[A-Za-z0-9+/=]+/gi, "[image]"); + } + + if (content && content.trim()) { + const ts = typeof m.timestamp === "number" ? m.timestamp : Date.now(); + result.push({ + id: (typeof m.id === "string" && m.id) ? m.id : generateMessageId(), + role: role as "user" | "assistant", + content: content.trim(), + timestamp: ts, + }); + } + } + + return result; +} + +/** + * Format local date as YYYY-MM-DD. + */ +function formatLocalDate(d: Date): string { + const y = d.getFullYear(); + const m = String(d.getMonth() + 1).padStart(2, "0"); + const day = String(d.getDate()).padStart(2, "0"); + return `${y}-${m}-${day}`; +} diff --git a/src/hooks/auto-capture.ts b/src/hooks/auto-capture.ts new file mode 100644 index 0000000..906a2ab --- /dev/null +++ b/src/hooks/auto-capture.ts @@ -0,0 +1,270 @@ +/** + * auto-capture hook (v3): records conversation messages locally (L0), + * then notifies the MemoryPipelineManager for L1/L2/L3 scheduling. + * + * Key design decisions: + * - Always write L0 locally via l0-recorder. + * - When VectorStore + EmbeddingService are available, also write L0 vector index. + * - Notify MemoryPipelineManager for L1/L2/L3 trigger evaluation. + * - L1 Runner reads from VectorStore DB (primary) or L0 JSONL files (fallback). + * - Extraction is NOT triggered here. The pipeline manager decides when. + */ + +import crypto from "node:crypto"; +import type { MemoryTdaiConfig } from "../config.js"; +import { CheckpointManager } from "../utils/checkpoint.js"; +import type { MemoryPipelineManager } from "../utils/pipeline-manager.js"; +import { recordConversation } from "../conversation/l0-recorder.js"; +import type { ConversationMessage } from "../conversation/l0-recorder.js"; +import type { VectorStore, L0VectorRecord } from "../store/vector-store.js"; +import type { EmbeddingService } from "../store/embedding.js"; + +const TAG = "[memory-tdai] [capture]"; + +interface Logger { + debug?: (message: string) => void; + info: (message: string) => void; + warn: (message: string) => void; + error: (message: string) => void; +} + +export interface AutoCaptureResult { + /** Whether the scheduler was notified (conversation count incremented) */ + schedulerNotified: boolean; + /** Number of messages recorded to L0 */ + l0RecordedCount: number; + /** Number of L0 message vectors written */ + l0VectorsWritten: number; + /** Filtered messages for L1 immediate use */ + filteredMessages: ConversationMessage[]; +} + +/** + * Generate a unique L0 record ID for vector indexing. + * Includes an index to distinguish multiple messages within the same round. + */ +function generateL0RecordId(sessionKey: string, index: number): string { + return `l0_${sessionKey}_${Date.now()}_${index}_${crypto.randomBytes(3).toString("hex")}`; +} + +export async function performAutoCapture(params: { + messages: unknown[]; + sessionKey: string; + sessionId?: string; + cfg: MemoryTdaiConfig; + pluginDataDir: string; + logger?: Logger; + scheduler?: MemoryPipelineManager; + /** Clean original user prompt from before_prompt_build cache (pre-prependContext). */ + originalUserText?: string; + /** + * Number of messages in the session at before_prompt_build time. + * Used by l0-recorder to locate the exact user message that originalUserText + * corresponds to: rawMessages[originalUserMessageCount] is the polluted user message. + */ + originalUserMessageCount?: number; + /** Epoch ms when the plugin was registered (cold-start time). + * Used as fallback cursor when checkpoint has no prior timestamp — + * prevents the first agent_end from dumping all session history into L0. */ + pluginStartTimestamp?: number; + /** VectorStore for L0 vector indexing (optional). */ + vectorStore?: VectorStore; + /** EmbeddingService for L0 vector indexing (optional). */ + embeddingService?: EmbeddingService; +}): Promise { + const { + messages, sessionKey, sessionId, cfg, pluginDataDir, logger, scheduler, + originalUserText, originalUserMessageCount, pluginStartTimestamp, + vectorStore, embeddingService, + } = params; + const tCaptureStart = performance.now(); + + const checkpoint = new CheckpointManager(pluginDataDir, logger); + + // ============================ + // Step 1 + 2: L0 recording + checkpoint update (ATOMIC) + // ============================ + // These steps are combined inside captureAtomically() to prevent the race + // condition where two concurrent agent_end events both read the same stale + // cursor and produce duplicate L0 records. The file lock is held for the + // entire read-cursor → recordConversation → advance-cursor sequence. + const tL0RecordStart = performance.now(); + let filteredMessages: ConversationMessage[] = []; + try { + await checkpoint.captureAtomically( + sessionKey, + pluginStartTimestamp, + async (afterTimestamp) => { + logger?.debug?.(`${TAG} L0 capture cursor (per-session, atomic): afterTimestamp=${afterTimestamp} session=${sessionKey}`); + + if (afterTimestamp === pluginStartTimestamp && pluginStartTimestamp && pluginStartTimestamp > 0) { + logger?.debug?.( + `${TAG} No per-session checkpoint cursor found for session=${sessionKey} — ` + + `using pluginStartTimestamp as floor: ` + + `${afterTimestamp} (${new Date(afterTimestamp).toISOString()})`, + ); + } + + filteredMessages = await recordConversation({ + sessionKey, + sessionId, + rawMessages: messages, + baseDir: pluginDataDir, + logger, + originalUserText, + afterTimestamp, + originalUserMessageCount, + }); + + if (filteredMessages.length === 0) { + return null; // Nothing captured — cursor stays unchanged + } + + logger?.debug?.(`${TAG} L0 recorded: ${filteredMessages.length} messages for session ${sessionKey}`); + const maxTs = Math.max(...filteredMessages.map((m) => m.timestamp)); + return { maxTimestamp: maxTs, messageCount: filteredMessages.length }; + }, + ); + } catch (err) { + logger?.error(`${TAG} L0 recording failed: ${err instanceof Error ? err.message : String(err)}`); + } + const tL0RecordEnd = performance.now(); + + // ============================ + // Step 1.5: L0 vector indexing — metadata written synchronously, + // embedding done in background (non-blocking) + // ============================ + // PERF FIX: Remote embedding API calls (2-3s each) were blocking + // the agent_end hook, adding 5-9s latency per conversation round. + // Now we: + // 1. Write L0 metadata + FTS immediately (no embedding) — ~10ms + // 2. Fire off background task to embed + update vectors (non-blocking) + // This way the user gets their response immediately. + const tL0VecStart = performance.now(); + let l0VectorsWritten = 0; + logger?.debug?.( + `${TAG} [L0-vec-index] Check: filteredMessages=${filteredMessages.length}, ` + + `vectorStore=${vectorStore ? "available" : "UNAVAILABLE"}, ` + + `embeddingService=${embeddingService ? "available" : "UNAVAILABLE"}`, + ); + + // Pre-generate L0 records and write metadata synchronously (fast path) + const l0Records: Array<{ record: L0VectorRecord; content: string }> = []; + if (filteredMessages.length > 0 && vectorStore) { + const now = new Date().toISOString(); + logger?.debug?.(`${TAG} [L0-vec-index] START indexing ${filteredMessages.length} message(s) for session ${sessionKey}`); + for (let i = 0; i < filteredMessages.length; i++) { + const msg = filteredMessages[i]; + try { + const l0Record: L0VectorRecord = { + id: generateL0RecordId(sessionKey, i), + sessionKey, + sessionId: sessionId || "", + role: msg.role, + messageText: msg.content, + recordedAt: now, + timestamp: msg.timestamp, + }; + + // Write metadata + FTS immediately WITHOUT embedding (fast, ~ms) + const upsertOk = vectorStore.upsertL0(l0Record, undefined); + if (upsertOk) { + l0VectorsWritten++; + l0Records.push({ record: l0Record, content: msg.content }); + } else { + logger?.warn(`${TAG} [L0-vec-index] upsertL0 returned false for message ${i}`); + } + } catch (err) { + logger?.warn?.(`${TAG} [L0-vec-index] FAILED for message ${i} (non-blocking): ${err instanceof Error ? err.message : String(err)}`); + } + } + logger?.debug?.(`${TAG} [L0-vec-index] DONE: ${l0VectorsWritten}/${filteredMessages.length} metadata records written (sync)`); + + // Fire-and-forget: batch embed + update vectors in background + if (l0Records.length > 0 && embeddingService) { + const bgVectorStore = vectorStore; // capture for closure + const bgEmbeddingService = embeddingService; + const bgRecords = [...l0Records]; // snapshot + const bgLogger = logger; + + // Do NOT await — this runs in background after response is sent + void (async () => { + const tBgStart = performance.now(); + try { + // Use embedBatch for a single API call instead of N sequential calls + const texts = bgRecords.map((r) => r.content); + const embeddings = await bgEmbeddingService.embedBatch(texts); + + let bgUpdated = 0; + for (let i = 0; i < bgRecords.length; i++) { + try { + const ok = bgVectorStore.updateL0Embedding(bgRecords[i].record.id, embeddings[i]); + if (ok) bgUpdated++; + } catch (err) { + bgLogger?.warn?.( + `${TAG} [L0-vec-index-bg] Failed to update embedding for ${bgRecords[i].record.id}: ` + + `${err instanceof Error ? err.message : String(err)}`, + ); + } + } + const bgMs = performance.now() - tBgStart; + bgLogger?.debug?.( + `${TAG} [L0-vec-index-bg] Background embedding complete: ${bgUpdated}/${bgRecords.length} vectors updated (${bgMs.toFixed(0)}ms)`, + ); + } catch (err) { + const bgMs = performance.now() - tBgStart; + bgLogger?.warn?.( + `${TAG} [L0-vec-index-bg] Background embedding failed (${bgMs.toFixed(0)}ms, non-fatal): ` + + `${err instanceof Error ? err.message : String(err)}`, + ); + } + })(); + } + } else if (filteredMessages.length > 0) { + logger?.warn(`${TAG} [L0-vec-index] SKIPPED: vectorStore not available`); + } + const tL0VecEnd = performance.now(); + + // ============================ + // Step 3: Notify scheduler of this conversation round + // ============================ + const tNotifyStart = performance.now(); + // Pass empty array: L1 Runner reads from VectorStore DB (or L0 JSONL fallback), not from in-memory buffers. + if (scheduler) { + await scheduler.notifyConversation(sessionKey, []); + logger?.debug?.(`${TAG} Scheduler notified of conversation round (sessionKey=${sessionKey})`); + + const totalMs = performance.now() - tCaptureStart; + logger?.info( + `${TAG} ⏱ Capture timing: total=${totalMs.toFixed(0)}ms, ` + + `l0Record+checkpoint=${(tL0RecordEnd - tL0RecordStart).toFixed(0)}ms, ` + + `l0VecIndex=${(tL0VecEnd - tL0VecStart).toFixed(0)}ms ` + + `(metadata-only, embed=background, msgs=${filteredMessages.length}), ` + + `notify=${(performance.now() - tNotifyStart).toFixed(0)}ms`, + ); + + return { + schedulerNotified: true, + l0RecordedCount: filteredMessages.length, + l0VectorsWritten, + filteredMessages, + }; + } + + const totalMs = performance.now() - tCaptureStart; + logger?.info( + `${TAG} ⏱ Capture timing: total=${totalMs.toFixed(0)}ms, ` + + `l0Record+checkpoint=${(tL0RecordEnd - tL0RecordStart).toFixed(0)}ms, ` + + `l0VecIndex=${(tL0VecEnd - tL0VecStart).toFixed(0)}ms ` + + `(metadata-only, embed=background, msgs=${filteredMessages.length}), ` + + `notify=${(performance.now() - tNotifyStart).toFixed(0)}ms`, + ); + + logger?.debug?.(`${TAG} No scheduler provided, skipping notification`); + return { + schedulerNotified: false, + l0RecordedCount: filteredMessages.length, + l0VectorsWritten, + filteredMessages, + }; +} diff --git a/src/hooks/auto-recall.ts b/src/hooks/auto-recall.ts new file mode 100644 index 0000000..f36222d --- /dev/null +++ b/src/hooks/auto-recall.ts @@ -0,0 +1,762 @@ +/** + * auto-recall hook (v3): injects relevant memories + persona into agent context + * before the agent starts processing. + * + * - Searches L1 memories using configurable strategy (keyword / embedding / hybrid) + * - keyword: FTS5 BM25 (requires FTS5; returns empty if unavailable) + * - embedding: VectorStore cosine similarity + * - hybrid: keyword + embedding merged with RRF + * - L3 persona injection + * - L2 scene navigation (full injection, LLM decides relevance) + */ + +import fs from "node:fs/promises"; +import path from "node:path"; +import type { MemoryTdaiConfig } from "../config.js"; +import { readSceneIndex } from "../scene/scene-index.js"; +import { generateSceneNavigation, stripSceneNavigation } from "../scene/scene-navigation.js"; +import type { MemoryRecord } from "../record/l1-reader.js"; +import type { VectorStore, VectorSearchResult, FtsSearchResult } from "../store/vector-store.js"; +import { buildFtsQuery } from "../store/vector-store.js"; +import type { EmbeddingService } from "../store/embedding.js"; +import { sanitizeText } from "../utils/sanitize.js"; + +const TAG = "[memory-tdai] [recall]"; + +/** + * Memory tools usage guide — injected at the end of memory context so the + * main agent knows how to actively retrieve deeper information. + */ +const MEMORY_TOOLS_GUIDE = ` +## 记忆工具调用指南 + +当上方注入的记忆片段不足以回答用户问题时，可主动调用以下工具获取更多信息： + +- **tdai_memory_search**：搜索结构化记忆（L1），适用于回忆用户偏好、历史事件节点、规则等关键信息。 +- **tdai_conversation_search**：搜索原始对话（L0），适用于查找具体消息原文、时间线、上下文细节；也可用于补充或校验 memory_search 的结果。 +- **read_file**（Scene Navigation 中的路径）：当已定位到相关情境，且需要该场景的完整画像、事件经过或阶段结论时使用。 +` + +/** + * Build the dynamic scene-navigation read_file hint. + * Tells the agent how to resolve relative paths in scene navigation + * by prepending the actual pluginDataDir. + */ +function buildScenePathHint(pluginDataDir: string): string { + return `⚠️ Scene Navigation 路径提示：上方 Scene Navigation 中的 Path（如 \`scene_blocks/xxx.md\`）是相对路径，使用 read_file 读取时需拼接为绝对路径：\`${pluginDataDir}/scene_blocks/xxx.md\``; +} + +interface Logger { + debug?: (message: string) => void; + info: (message: string) => void; + warn: (message: string) => void; + error: (message: string) => void; +} + +/** A single recalled L1 memory with its search score and type. */ +export interface RecalledMemory { + content: string; + score: number; + type: string; +} + +export interface RecallResult { + /** Injected before user message (prepended to the user's prompt text by openclaw) */ + prependContext?: string; + /** Appended to system prompt (all memory context: persona, scene navigation, relevant memories) */ + appendSystemContext?: string; + + // ── Metric payload (for pendingRecallCache in index.ts) ── + /** L1 memories that were recalled (with scores), for metric reporting */ + recalledL1Memories?: RecalledMemory[]; + /** L3 Persona raw content loaded during recall (null if none) */ + recalledL3Persona?: string | null; + /** Effective search strategy used */ + recallStrategy?: string; +} + +export async function performAutoRecall(params: { + userText: string; + actorId: string; + sessionKey: string; + cfg: MemoryTdaiConfig; + pluginDataDir: string; + logger?: Logger; + vectorStore?: VectorStore; + embeddingService?: EmbeddingService; +}): Promise { + const { cfg, logger } = params; + const timeoutMs = cfg.recall.timeoutMs ?? 5000; + + let timer: ReturnType | undefined; + + return Promise.race([ + performAutoRecallInner(params).finally(() => { + if (timer) clearTimeout(timer); + }), + new Promise((resolve) => { + timer = setTimeout(() => { + logger?.warn?.( + `${TAG} ⚠️ Recall timed out after ${timeoutMs}ms — skipping memory injection to avoid blocking the user`, + ); + resolve(undefined); + }, timeoutMs); + }), + ]); +} + +async function performAutoRecallInner(params: { + userText: string; + actorId: string; + sessionKey: string; + cfg: MemoryTdaiConfig; + pluginDataDir: string; + logger?: Logger; + vectorStore?: VectorStore; + embeddingService?: EmbeddingService; +}): Promise { + const { userText, cfg, pluginDataDir, logger, vectorStore, embeddingService } = params; + const tRecallStart = performance.now(); + + // Search relevant memories (L1 layer) — skip only when userText is empty/undefined + const tSearchStart = performance.now(); + let memoryLines: string[] = []; + let effectiveStrategy = "skipped"; + let recalledL1Memories: RecalledMemory[] = []; + let searchTiming: SearchTiming = { ftsMs: 0, embeddingMs: 0, ftsHits: 0, embeddingHits: 0 }; + if (!userText || userText.length === 0) { + logger?.debug?.(`${TAG} User text empty/undefined, skipping memory search (persona/scene still injected)`); + } else { + effectiveStrategy = cfg.recall.strategy ?? "hybrid"; + const searchResult = await searchMemories(userText, pluginDataDir, cfg, logger, effectiveStrategy as "keyword" | "embedding" | "hybrid", vectorStore, embeddingService); + memoryLines = searchResult.lines; + searchTiming = searchResult.timing; + + // Extract structured RecalledMemory from formatted lines for metric reporting + recalledL1Memories = memoryLines.map((line) => { + const match = line.match(/^-\s+\[([^\]]+)\]\s+(.+?)(?:\s*\(活动时间:.*\))?$/); + if (match) { + const tag = match[1]; + const content = match[2].trim(); + const typePart = tag.includes("|") ? tag.split("|")[0] : tag; + return { content, score: 0, type: typePart }; + } + return { content: line, score: 0, type: "unknown" }; + }); + } + const tSearchEnd = performance.now(); + + // Read persona (L3 layer) + const tPersonaStart = performance.now(); + let personaContent: string | undefined; + try { + const personaPath = path.join(pluginDataDir, "persona.md"); + const raw = await fs.readFile(personaPath, "utf-8"); + personaContent = stripSceneNavigation(raw).trim(); + if (!personaContent) personaContent = undefined; + logger?.debug?.(`${TAG} Persona loaded: ${personaContent ? `${personaContent.length} chars` : "empty"}`); + } catch { + logger?.debug?.(`${TAG} No persona file found (expected for new users)`); + } + const tPersonaEnd = performance.now(); + + // Load full scene navigation (L2 layer) + const tSceneStart = performance.now(); + let sceneNavigation: string | undefined; + try { + const sceneIndex = await readSceneIndex(pluginDataDir); + if (sceneIndex.length > 0) { + sceneNavigation = generateSceneNavigation(sceneIndex); + logger?.debug?.(`${TAG} Scene navigation generated: ${sceneIndex.length} scenes`); + } + } catch { + logger?.debug?.(`${TAG} No scene index found`); + } + const tSceneEnd = performance.now(); + + if (memoryLines.length === 0 && !personaContent && !sceneNavigation) { + const totalMs = performance.now() - tRecallStart; + logger?.info( + `${TAG} ⏱ Recall timing: total=${totalMs.toFixed(0)}ms, ` + + `search=${(tSearchEnd - tSearchStart).toFixed(0)}ms(strategy=${effectiveStrategy},hits=${memoryLines.length},` + + `fts=${searchTiming.ftsMs.toFixed(0)}ms/${searchTiming.ftsHits}hits,` + + `vec=${searchTiming.embeddingMs.toFixed(0)}ms/${searchTiming.embeddingHits}hits), ` + + `persona=${(tPersonaEnd - tPersonaStart).toFixed(0)}ms, ` + + `scene=${(tSceneEnd - tSceneStart).toFixed(0)}ms — no context to inject`, + ); + logger?.debug?.(`${TAG} No memories/persona/scenes to inject`); + return undefined; + } + + // All memory context → appendSystemContext (system prompt end) + // Order: user-persona → scene-navigation → relevant-memories → tools-guide + const systemParts: string[] = []; + if (personaContent) { + systemParts.push(`\n${personaContent}\n`); + } + if (sceneNavigation) { + const pathHint = buildScenePathHint(pluginDataDir); + systemParts.push(`\n${sceneNavigation}\n\n${pathHint}\n`); + } + if (memoryLines.length > 0) { + systemParts.push(`\n${memoryLines.join("\n")}\n`); + } + + // Append memory tools usage guide so the agent knows how to actively + // retrieve deeper context when the injected snippets are not enough. + if (systemParts.length > 0) { + systemParts.push(MEMORY_TOOLS_GUIDE); + } + + const appendSystemContext = systemParts.length > 0 ? systemParts.join("\n\n") : undefined; + + const totalMs = performance.now() - tRecallStart; + logger?.info( + `${TAG} ⏱ Recall timing: total=${totalMs.toFixed(0)}ms, ` + + `search=${(tSearchEnd - tSearchStart).toFixed(0)}ms(strategy=${effectiveStrategy},hits=${memoryLines.length},` + + `fts=${searchTiming.ftsMs.toFixed(0)}ms/${searchTiming.ftsHits}hits,` + + `vec=${searchTiming.embeddingMs.toFixed(0)}ms/${searchTiming.embeddingHits}hits), ` + + `persona=${(tPersonaEnd - tPersonaStart).toFixed(0)}ms(${personaContent ? `${personaContent.length}chars` : "none"}), ` + + `scene=${(tSceneEnd - tSceneStart).toFixed(0)}ms(${sceneNavigation ? "loaded" : "none"})`, + ); + + if (!appendSystemContext) { + return undefined; + } + + return { + appendSystemContext, + recalledL1Memories, + recalledL3Persona: personaContent ?? null, + recallStrategy: effectiveStrategy, + }; +} + +// ============================ +// Multi-strategy search dispatcher +// ============================ + +interface ScoredRecord { + record: MemoryRecord; + score: number; +} + +/** Timing breakdown from memory search */ +interface SearchTiming { + ftsMs: number; + embeddingMs: number; + ftsHits: number; + embeddingHits: number; +} + +interface SearchResult { + lines: string[]; + timing: SearchTiming; +} + +/** + * Search memories and return both formatted lines and structured details. + * + * This is a thin wrapper around `searchMemories` that also captures + * the recalled memory metadata for metric reporting (agent_turn event). + * It parses the returned formatted lines to extract type/content info. + */ +async function searchMemoriesWithDetails( + userText: string, + pluginDataDir: string, + cfg: MemoryTdaiConfig, + logger: Logger | undefined, + strategy: "keyword" | "embedding" | "hybrid", + vectorStore?: VectorStore, + embeddingService?: EmbeddingService, +): Promise<{ lines: string[]; memories: RecalledMemory[]; timing: SearchTiming }> { + const result = await searchMemories(userText, pluginDataDir, cfg, logger, strategy, vectorStore, embeddingService); + + // Extract structured data from formatted memory lines. + // Format: "- [type|scene] content (活动时间: ...)" or "- [type] content" + const memories: RecalledMemory[] = result.lines.map((line) => { + const match = line.match(/^-\s+\[([^\]]+)\]\s+(.+?)(?:\s*\(活动时间:.*\))?$/); + if (match) { + const tag = match[1]; + const content = match[2].trim(); + const typePart = tag.includes("|") ? tag.split("|")[0] : tag; + return { content, score: 0, type: typePart }; + } + return { content: line, score: 0, type: "unknown" }; + }); + + return { lines: result.lines, memories, timing: result.timing }; +} + +/** + * Search memories using the configured strategy. + * + * - "keyword": JSONL keyword-based (Jaccard similarity) — no embedding needed + * - "embedding": VectorStore cosine similarity — requires vectorStore + embeddingService + * - "hybrid": merge both keyword and embedding results with RRF (Reciprocal Rank Fusion) + * + * Falls back to keyword if embedding resources are unavailable. + */ +async function searchMemories( + userText: string, + pluginDataDir: string, + cfg: MemoryTdaiConfig, + logger: Logger | undefined, + strategy: "keyword" | "embedding" | "hybrid", + vectorStore?: VectorStore, + embeddingService?: EmbeddingService, +): Promise { + const emptyResult: SearchResult = { lines: [], timing: { ftsMs: 0, embeddingMs: 0, ftsHits: 0, embeddingHits: 0 } }; + // Strip gateway-injected inbound metadata (Sender, timestamps, media markers, + // base64 image data, etc.) so FTS / embedding queries are based on pure user intent. + const cleanText = sanitizeText(userText); + + if (cleanText.length < 2) { + logger?.debug?.(`${TAG} Query too short for memory search (raw=${userText.length}, clean=${cleanText.length})`); + return emptyResult; + } + + if (cleanText.length !== userText.length) { + logger?.debug?.( + `${TAG} userText sanitized: ${userText.length} → ${cleanText.length} chars`, + ); + } + + const maxResults = cfg.recall.maxResults ?? 5; + const threshold = cfg.recall.scoreThreshold ?? 0.3; + + const embeddingAvailable = !!vectorStore && !!embeddingService; + + logger?.debug?.( + `${TAG} [searchMemories] strategy=${strategy}, embeddingAvailable=${embeddingAvailable}, ` + + `vectorStore=${vectorStore ? "available" : "UNAVAILABLE"}, ` + + `embeddingService=${embeddingService ? "available" : "UNAVAILABLE"}, ` + + `maxResults=${maxResults}, threshold=${threshold}`, + ); + + // Determine effective strategy (fall back to keyword if embedding not available) + let effectiveStrategy = strategy; + if ((strategy === "embedding" || strategy === "hybrid") && !embeddingAvailable) { + logger?.warn?.( + `${TAG} Strategy "${strategy}" requested but EmbeddingService not available, falling back to keyword`, + ); + effectiveStrategy = "keyword"; + } + + logger?.debug?.(`${TAG} Search strategy: ${effectiveStrategy} (configured: ${strategy})`); + + try { + if (effectiveStrategy === "keyword") { + const tFts = performance.now(); + const lines = await searchByKeyword(cleanText, pluginDataDir, maxResults, threshold, logger, vectorStore); + return { lines, timing: { ftsMs: performance.now() - tFts, embeddingMs: 0, ftsHits: lines.length, embeddingHits: 0 } }; + } + + if (effectiveStrategy === "embedding") { + const tEmb = performance.now(); + const lines = await searchByEmbedding(cleanText, maxResults, threshold, vectorStore!, embeddingService!, logger); + return { lines, timing: { ftsMs: 0, embeddingMs: performance.now() - tEmb, ftsHits: 0, embeddingHits: lines.length } }; + } + + // Hybrid: run both keyword and embedding, merge with RRF + return await searchHybrid(cleanText, pluginDataDir, maxResults, threshold, vectorStore!, embeddingService!, logger); + } catch (err) { + logger?.warn?.(`${TAG} Memory search failed (strategy=${effectiveStrategy}): ${err instanceof Error ? err.message : String(err)}`); + return emptyResult; + } +} + +// ============================ +// Strategy: Keyword (FTS5 BM25, no in-memory fallback) +// ============================ + +async function searchByKeyword( + userText: string, + _pluginDataDir: string, + maxResults: number, + threshold: number, + logger?: Logger, + vectorStore?: VectorStore, +): Promise { + // Prefer FTS5 if available + if (vectorStore?.isFtsAvailable()) { + const ftsQuery = buildFtsQuery(userText); + if (ftsQuery) { + logger?.debug?.(`${TAG} [keyword-fts] Using FTS5 BM25 search: query="${ftsQuery}"`); + const ftsResults = vectorStore.ftsSearchL1(ftsQuery, maxResults * 2); + if (ftsResults.length > 0) { + logger?.debug?.( + `${TAG} [keyword-fts] FTS5 raw results (${ftsResults.length}): ` + + ftsResults.map((r) => `id=${r.record_id} score=${r.score.toFixed(6)}`).join(", "), + ); + const filtered = ftsResults + .filter((r) => r.score >= threshold) + .slice(0, maxResults); + + if (filtered.length > 0) { + logger?.debug?.(`${TAG} [keyword-fts] FTS5 found ${filtered.length} results (from ${ftsResults.length} raw, threshold=${threshold})`); + return filtered.map((r) => formatMemoryLine(ftsResultToFormatable(r))); + } + + // BM25 absolute scores are unreliable when the document set is very + // small (e.g. 1–3 records) because IDF approaches 0. In that case, + // trust FTS5's MATCH + rank ordering and return the top results anyway. + if (ftsResults.length <= maxResults) { + logger?.debug?.( + `${TAG} [keyword-fts] All ${ftsResults.length} results below threshold=${threshold} ` + + `but document set is small — returning all matched results`, + ); + return ftsResults.slice(0, maxResults).map((r) => formatMemoryLine(ftsResultToFormatable(r))); + } + logger?.debug?.(`${TAG} [keyword-fts] FTS5 returned 0 results above threshold (from ${ftsResults.length} raw)`); + } + } + } + + // FTS5 not available or returned no results — skip in-memory fallback to avoid O(N) full scan + logger?.debug?.(`${TAG} [keyword] FTS5 unavailable or no results, skipping keyword search`); + return []; +} + +// ============================ +// Strategy: Embedding (VectorStore cosine) +// ============================ + +async function searchByEmbedding( + userText: string, + maxResults: number, + threshold: number, + vectorStore: VectorStore, + embeddingService: EmbeddingService, + logger?: Logger, +): Promise { + logger?.debug?.( + `${TAG} [embedding-search] START query="${userText.slice(0, 80)}...", maxResults=${maxResults}, threshold=${threshold}`, + ); + const queryEmbedding = await embeddingService.embed(userText); + logger?.debug?.( + `${TAG} [embedding-search] Query embedding OK: dims=${queryEmbedding.length}, ` + + `norm=${Math.sqrt(Array.from(queryEmbedding).reduce((s, v) => s + v * v, 0)).toFixed(4)}, ` + + `searching top-${maxResults * 2}...`, + ); + // Retrieve more candidates for subsequent filtering + const vecResults: VectorSearchResult[] = vectorStore.search(queryEmbedding, maxResults * 2); + + if (vecResults.length === 0) { + logger?.debug?.(`${TAG} [embedding-search] Returned 0 results`); + return []; + } + + logger?.debug?.(`${TAG} [embedding-search] Got ${vecResults.length} candidates, filtering by threshold=${threshold}`); + for (const r of vecResults) { + logger?.debug?.( + `${TAG} [embedding-search] candidate id=${r.record_id}, score=${r.score.toFixed(4)}, ` + + `type=${r.type}, content="${r.content.slice(0, 60)}..."`, + ); + } + + const filtered = vecResults + .filter((r) => r.score >= threshold) + .slice(0, maxResults); + + if (filtered.length > 0) { + logger?.debug?.(`${TAG} [embedding-search] Found ${filtered.length} relevant memories above threshold (from ${vecResults.length} candidates)`); + return filtered.map((r) => formatMemoryLine(vectorResultToFormatable(r))); + } + + logger?.debug?.(`${TAG} [embedding-search] No results above threshold ${threshold}`); + return []; +} + +// ============================ +// Strategy: Hybrid (Keyword + Embedding + RRF) +// ============================ + +/** + * Hybrid search: run keyword (FTS5) and embedding in parallel, merge with + * Reciprocal Rank Fusion (RRF) to combine rank lists. + * + * RRF score for a record at rank r = 1 / (k + r), where k=60 is a constant. + * If a record appears in both lists, its RRF scores are summed. + * + * If FTS5 is unavailable, the keyword side returns empty and RRF uses + * embedding results only. + */ +async function searchHybrid( + userText: string, + _pluginDataDir: string, + maxResults: number, + _threshold: number, + vectorStore: VectorStore, + embeddingService: EmbeddingService, + logger?: Logger, +): Promise { + // Run keyword and embedding searches in parallel + const candidateK = maxResults * 3; // retrieve more for merging + + const [keywordResult, embeddingResult] = await Promise.all([ + // Keyword search: FTS5 only (no in-memory fallback) + (async () => { + const tStart = performance.now(); + try { + // Try FTS5 first + if (vectorStore.isFtsAvailable()) { + const ftsQuery = buildFtsQuery(userText); + if (ftsQuery) { + const ftsResults = vectorStore.ftsSearchL1(ftsQuery, candidateK); + if (ftsResults.length > 0) { + logger?.debug?.(`${TAG} [hybrid-keyword-fts] FTS5 found ${ftsResults.length} candidates`); + // Convert FtsSearchResult to ScoredRecord for RRF merge + const records = ftsResults.map((r): ScoredRecord => ({ + record: { + id: r.record_id, + content: r.content, + type: r.type as MemoryRecord["type"], + priority: r.priority, + scene_name: r.scene_name, + source_message_ids: [], + metadata: r.metadata_json ? (() => { try { return JSON.parse(r.metadata_json); } catch { return {}; } })() : {}, + timestamps: [r.timestamp_str].filter(Boolean), + createdAt: "", + updatedAt: "", + sessionKey: r.session_key, + sessionId: r.session_id, + }, + score: r.score, + })); + return { records, ms: performance.now() - tStart }; + } + } + } + // FTS5 not available or returned no results — skip in-memory fallback + logger?.debug?.(`${TAG} [hybrid-keyword] FTS5 unavailable or no results, skipping keyword part`); + return { records: [] as ScoredRecord[], ms: performance.now() - tStart }; + } catch (err) { + logger?.warn?.(`${TAG} Hybrid: keyword part failed: ${err instanceof Error ? err.message : String(err)}`); + return { records: [] as ScoredRecord[], ms: performance.now() - tStart }; + } + })(), + // Embedding search + (async () => { + const tStart = performance.now(); + try { + logger?.debug?.(`${TAG} [hybrid-embedding] Generating query embedding...`); + const queryEmbedding = await embeddingService.embed(userText); + logger?.debug?.( + `${TAG} [hybrid-embedding] Embedding OK, dims=${queryEmbedding.length}, searching top-${candidateK}...`, + ); + const results = vectorStore.search(queryEmbedding, candidateK); + logger?.debug?.(`${TAG} [hybrid-embedding] Got ${results.length} candidates`); + return { results, ms: performance.now() - tStart }; + } catch (err) { + logger?.warn?.(`${TAG} Hybrid: embedding part failed: ${err instanceof Error ? err.message : String(err)}`); + return { results: [] as VectorSearchResult[], ms: performance.now() - tStart }; + } + })(), + ]); + + const keywordResults = keywordResult.records; + const embeddingResults = embeddingResult.results; + const timing: SearchTiming = { + ftsMs: keywordResult.ms, + embeddingMs: embeddingResult.ms, + ftsHits: keywordResults.length, + embeddingHits: embeddingResults.length, + }; + + if (keywordResults.length === 0 && embeddingResults.length === 0) { + logger?.debug?.(`${TAG} Hybrid search: both strategies returned 0 results`); + return { lines: [], timing }; + } + + // RRF merge: k=60 is a standard constant from the RRF paper + const RRF_K = 60; + + // Map: record_id → { rrfScore, formatable } + const mergedMap = new Map(); + + // Process keyword results + for (let rank = 0; rank < keywordResults.length; rank++) { + const r = keywordResults[rank]; + const id = r.record.id; + const rrfScore = 1 / (RRF_K + rank + 1); + const existing = mergedMap.get(id); + if (existing) { + existing.rrfScore += rrfScore; + } else { + mergedMap.set(id, { rrfScore, formatable: recordToFormatable(r.record) }); + } + } + + // Process embedding results + for (let rank = 0; rank < embeddingResults.length; rank++) { + const r = embeddingResults[rank]; + const id = r.record_id; + const rrfScore = 1 / (RRF_K + rank + 1); + const existing = mergedMap.get(id); + if (existing) { + existing.rrfScore += rrfScore; + } else { + mergedMap.set(id, { rrfScore, formatable: vectorResultToFormatable(r) }); + } + } + + // Sort by combined RRF score and take top results + const sorted = [...mergedMap.entries()] + .sort((a, b) => b[1].rrfScore - a[1].rrfScore) + .slice(0, maxResults); + + if (sorted.length > 0) { + logger?.debug?.( + `${TAG} Hybrid search found ${sorted.length} results ` + + `(keyword=${keywordResults.length}, embedding=${embeddingResults.length})`, + ); + return { lines: sorted.map(([, { formatable }]) => formatMemoryLine(formatable)), timing }; + } + + logger?.debug?.(`${TAG} Hybrid search: no results after merge`); + return { lines: [], timing }; +} + +// ============================ +// Unified memory line formatter +// ============================ + +/** + * Format a single memory record into a rich natural-language line for prompt injection. + * + * Time semantics: + * - timestamp (点时间): when the activity/event happened, e.g. "2025-03-01 mentioned something" + * - activity_start_time / activity_end_time (段时间): activity time range, e.g. "trip from 2025-05-01 to 2025-05-10" + * - All three time fields may be empty/undefined — handled gracefully. + * + * Output examples: + * - [persona] 用户叫王小明，30岁，是一名软件工程师。 + * - [episodic|旅行计划] 用户计划五月去日本旅行。(活动时间: 2025-05-01 ~ 2025-05-10) + * - [episodic] 用户今天加班到很晚。(活动时间: 2025-03-01) + * - [instruction] 用户要求回答时使用中文，保持简洁。 + */ +interface FormatableMemory { + type: string; + content: string; + scene_name?: string; + /** Activity time range start (段时间 start), may be empty */ + activity_start_time?: string; + /** Activity time range end (段时间 end), may be empty */ + activity_end_time?: string; + /** Activity point-in-time (点时间: when it happened), may be empty */ + timestamp?: string; +} + +function formatMemoryLine(m: FormatableMemory): string { + // 1. Type tag + optional scene name + const tag = m.scene_name ? `${m.type}|${m.scene_name}` : m.type; + + // 2. Content (core) + let line = `- [${tag}] ${m.content}`; + + // 3. Time info — prefer activity_start/end range; fall back to timestamp as point-in-time + const start = formatTimestamp(m.activity_start_time); + const end = formatTimestamp(m.activity_end_time); + const point = formatTimestamp(m.timestamp); + + if (start && end) { + // 段时间: both start and end + line += ` (活动时间: ${start} ~ ${end})`; + } else if (start) { + // 段时间: only start + line += ` (活动时间: ${start}起)`; + } else if (end) { + // 段时间: only end + line += ` (活动时间: 至${end})`; + } else if (point) { + // 点时间: single timestamp + line += ` (活动时间: ${point})`; + } + // If all three are empty → no time info appended (graceful) + + return line; +} + +/** + * Format an ISO 8601 timestamp to a concise date or datetime string. + * - If the time part is 00:00:00 → show date only (e.g. "2025-03-01") + * - Otherwise → show date + time (e.g. "2025-03-01 14:30") + * - Returns undefined for empty/invalid inputs. + */ +function formatTimestamp(ts: string | undefined): string | undefined { + if (!ts) return undefined; + // Try to parse ISO format: "2025-03-01T14:30:00.000Z" or "2025-03-01" + const match = ts.match(/^(\d{4}-\d{2}-\d{2})(?:T(\d{2}:\d{2})(?::\d{2})?)?/); + if (!match) return undefined; + const datePart = match[1]; + const timePart = match[2]; + if (!timePart || timePart === "00:00") { + return datePart; + } + return `${datePart} ${timePart}`; +} + +/** + * Build a FormatableMemory from a full MemoryRecord (keyword search path). + * Handles empty metadata, empty timestamps array gracefully. + */ +function recordToFormatable(record: MemoryRecord): FormatableMemory { + const meta = record.metadata as { activity_start_time?: string; activity_end_time?: string } | undefined; + return { + type: record.type, + content: record.content, + scene_name: record.scene_name || undefined, + activity_start_time: meta?.activity_start_time || undefined, + activity_end_time: meta?.activity_end_time || undefined, + timestamp: (record.timestamps && record.timestamps.length > 0) ? record.timestamps[0] : undefined, + }; +} + +/** + * Build a FormatableMemory from a VectorSearchResult (embedding search path). + * Handles empty/invalid metadata_json, empty timestamp_str gracefully. + */ +function vectorResultToFormatable(r: VectorSearchResult): FormatableMemory { + let activityStart: string | undefined; + let activityEnd: string | undefined; + if (r.metadata_json && r.metadata_json !== "{}") { + try { + const meta = typeof r.metadata_json === "string" ? JSON.parse(r.metadata_json) : r.metadata_json; + activityStart = meta?.activity_start_time || undefined; + activityEnd = meta?.activity_end_time || undefined; + } catch { /* ignore parse errors — treat as no metadata */ } + } + return { + type: r.type, + content: r.content, + scene_name: r.scene_name || undefined, + activity_start_time: activityStart, + activity_end_time: activityEnd, + timestamp: r.timestamp_str || undefined, + }; +} + +/** + * Build a FormatableMemory from an FtsSearchResult (FTS5 keyword search path). + * Handles empty/invalid metadata_json, empty timestamp_str gracefully. + */ +function ftsResultToFormatable(r: FtsSearchResult): FormatableMemory { + let activityStart: string | undefined; + let activityEnd: string | undefined; + if (r.metadata_json && r.metadata_json !== "{}") { + try { + const meta = typeof r.metadata_json === "string" ? JSON.parse(r.metadata_json) : r.metadata_json; + activityStart = meta?.activity_start_time || undefined; + activityEnd = meta?.activity_end_time || undefined; + } catch { /* ignore parse errors — treat as no metadata */ } + } + return { + type: r.type, + content: r.content, + scene_name: r.scene_name || undefined, + activity_start_time: activityStart, + activity_end_time: activityEnd, + timestamp: r.timestamp_str || undefined, + }; +} diff --git a/src/persona/persona-generator.ts b/src/persona/persona-generator.ts new file mode 100644 index 0000000..6f6481b --- /dev/null +++ b/src/persona/persona-generator.ts @@ -0,0 +1,205 @@ +/** + * PersonaGenerator: generates or updates user persona using the four-layer + * deep scan model via CleanContextRunner. + */ + +import fs from "node:fs/promises"; +import path from "node:path"; +import { CleanContextRunner } from "../utils/clean-context-runner.js"; +import { CheckpointManager } from "../utils/checkpoint.js"; +import { readSceneIndex } from "../scene/scene-index.js"; +import { generateSceneNavigation, stripSceneNavigation } from "../scene/scene-navigation.js"; +import { buildPersonaPrompt } from "../prompts/persona-generation.js"; +import { BackupManager } from "../utils/backup.js"; +import { escapeXmlTags } from "../utils/sanitize.js"; +import { report } from "../report/reporter.js"; + +const TAG = "[memory-tdai] [persona]"; + +interface Logger { + debug?: (message: string) => void; + info: (message: string) => void; + warn: (message: string) => void; + error: (message: string) => void; +} + +export class PersonaGenerator { + private dataDir: string; + private runner: CleanContextRunner; + private logger: Logger | undefined; + private backupCount: number; + private instanceId: string | undefined; + + constructor(opts: { + dataDir: string; + config: unknown; + model?: string; + backupCount?: number; + logger?: Logger; + /** Plugin instance ID for metric reporting (optional) */ + instanceId?: string; + }) { + this.dataDir = opts.dataDir; + this.logger = opts.logger; + this.backupCount = opts.backupCount ?? 3; + this.instanceId = opts.instanceId; + this.runner = new CleanContextRunner({ + config: opts.config, + modelRef: opts.model, + enableTools: true, + logger: opts.logger, + }); + this.logger?.debug?.(`${TAG} Generator created: model=${opts.model ?? "(default)"}, dataDir=${opts.dataDir}`); + } + + /** + * Execute persona generation. + */ + async generate(triggerReason?: string): Promise { + const startMs = Date.now(); + this.logger?.debug?.(`${TAG} Starting generation: reason="${triggerReason ?? "none"}"`); + + const cpManager = new CheckpointManager(this.dataDir); + const cp = await cpManager.read(); + this.logger?.debug?.(`${TAG} Checkpoint: total_processed=${cp.total_processed}, last_persona_at=${cp.last_persona_at}`); + + const personaPath = path.join(this.dataDir, "persona.md"); + + // 1. Read existing persona (strip navigation) + let existingPersona: string | undefined; + try { + const raw = await fs.readFile(personaPath, "utf-8"); + existingPersona = stripSceneNavigation(raw).trim() || undefined; + this.logger?.debug?.(`${TAG} Existing persona: ${existingPersona ? `${existingPersona.length} chars` : "empty"}`); + } catch { + this.logger?.debug?.(`${TAG} No existing persona file`); + } + + // 2. Load scene index + identify changed scenes + const index = await readSceneIndex(this.dataDir); + const changedScenes = index.filter((e) => { + if (!cp.last_persona_time) return true; + const updatedMs = new Date(e.updated).getTime(); + const personaMs = new Date(cp.last_persona_time).getTime(); + // If either date is unparseable (NaN), treat as changed (conservative) + if (Number.isNaN(updatedMs) || Number.isNaN(personaMs)) return true; + return updatedMs > personaMs; + }); + this.logger?.debug?.(`${TAG} Scene index: ${index.length} total, ${changedScenes.length} changed since last persona`); + + // 3. Read changed scene contents (full raw content including META, matching Python reference) + const blocksDir = path.join(this.dataDir, "scene_blocks"); + const changedSceneContents: string[] = []; + for (const entry of changedScenes) { + try { + const raw = await fs.readFile(path.join(blocksDir, entry.filename), "utf-8"); + changedSceneContents.push( + `### [${changedSceneContents.length + 1}] ${entry.filename}\n\n\`\`\`markdown\n${raw}\n\`\`\``, + ); + } catch { + this.logger?.warn(`${TAG} Could not read scene block: ${entry.filename}`); + } + } + + if (changedSceneContents.length === 0 && existingPersona) { + this.logger?.debug?.(`${TAG} No scene changes and persona exists, skipping generation`); + return false; + } + + // 4. Determine mode + const mode = existingPersona ? "incremental" : "first"; + this.logger?.debug?.(`${TAG} Generation mode: ${mode}, ${changedSceneContents.length} scene blocks to process`); + + // 5. Build changed scenes section with guidance (matching Python reference format) + let changedScenesContent: string; + if (changedSceneContents.length > 0) { + changedScenesContent = + `\n\n## 📄 变化场景完整内容\n\n` + + `*自上次 Persona 更新后，以下 ${changedSceneContents.length} 个场景发生了变化。工程已为你预加载完整内容：*\n\n` + + changedSceneContents.join("\n\n") + + `\n\n---\n\n` + + `⚠️ **重点分析变化场景**：上述场景是自上次更新后的**新增/修改内容**，请**重点分析**这些场景中的新信息。\n`; + } else { + changedScenesContent = `\n\n⚠️ **无变化场景**：所有场景均已在上次 Persona 更新中分析过，本次可直接读取所有场景进行全局审视。\n`; + } + + // 6. Build prompt + const prompt = buildPersonaPrompt({ + mode, + currentTime: new Date().toISOString(), + totalProcessed: cp.total_processed, + sceneCount: index.length, + changedSceneCount: changedScenes.length, + changedScenesContent, + existingPersona, + triggerInfo: triggerReason, + personaFilePath: personaPath, + checkpointPath: path.join(this.dataDir, ".metadata", "recall_checkpoint.json"), + }); + + // 7. Backup before LLM run (LLM writes persona.md via tools) + const bm = new BackupManager(path.join(this.dataDir, ".backup")); + await bm.backupFile(personaPath, "persona", `offset${cp.total_processed}`, this.backupCount); + + // 8. Run LLM agent (sandboxed to dataDir, tools enabled — LLM writes persona.md directly) + try { + this.logger?.debug?.(`${TAG} Calling LLM for persona generation (timeout=180s, tools=enabled, workspaceDir=${this.dataDir})...`); + await this.runner.run({ + prompt, + taskId: "persona-generation", + timeoutMs: 180_000, + // maxTokens omitted → core uses the resolved model's maxTokens from catalog + workspaceDir: this.dataDir, + }); + this.logger?.debug?.(`${TAG} LLM runner completed`); + } catch (err) { + const elapsedMs = Date.now() - startMs; + this.logger?.error(`${TAG} Persona generation failed after ${elapsedMs}ms: ${err instanceof Error ? err.stack ?? err.message : String(err)}`); + return false; + } + + // 9. Read LLM-written persona.md and apply post-processing + let personaText: string; + try { + personaText = await fs.readFile(personaPath, "utf-8"); + } catch { + // LLM failed to write persona.md — treat as failure + this.logger?.error(`${TAG} LLM did not write persona.md — file not found after runner completed`); + return false; + } + + // 10. Strip any navigation the LLM might have added + sanitize for safe injection + personaText = escapeXmlTags(stripSceneNavigation(personaText).trim()); + + if (!personaText) { + this.logger?.error(`${TAG} LLM wrote empty persona.md — skipping`); + return false; + } + + // 11. Append fresh scene navigation and write final content + const nav = generateSceneNavigation(index); + const finalContent = nav ? `${personaText}\n\n${nav}\n` : personaText; + await fs.writeFile(personaPath, finalContent, "utf-8"); + + // 12. Update checkpoint + await cpManager.markPersonaGenerated(cp.total_processed); + + const elapsedMs = Date.now() - startMs; + this.logger?.info(`${TAG} Persona written (${finalContent.length} chars) in ${elapsedMs}ms`); + + // ── l3_persona_generation metric ── + if (this.instanceId && this.logger) { + report("l3_persona_generation", { + triggerReason: triggerReason ?? "unknown", + mode: existingPersona ? "incremental" : "initial", + newPersonaContent: personaText, + newPersonaLength: personaText.length, + totalDurationMs: elapsedMs, + success: true, + error: null, + }); + } + + return true; + } +} diff --git a/src/persona/persona-trigger.ts b/src/persona/persona-trigger.ts new file mode 100644 index 0000000..250db01 --- /dev/null +++ b/src/persona/persona-trigger.ts @@ -0,0 +1,121 @@ +/** + * PersonaTrigger: determines whether to trigger persona generation. + * Implements the 5 trigger conditions from the legacy system. + */ + +import fs from "node:fs/promises"; +import path from "node:path"; +import { CheckpointManager } from "../utils/checkpoint.js"; +import { stripSceneNavigation } from "../scene/scene-navigation.js"; + +const TAG = "[memory-tdai] [trigger]"; + +interface TriggerLogger { + debug?: (message: string) => void; + info: (message: string) => void; + warn: (message: string) => void; + error: (message: string) => void; +} + +export interface TriggerResult { + should: boolean; + reason: string; +} + +export class PersonaTrigger { + private dataDir: string; + private interval: number; + private logger: TriggerLogger | undefined; + + constructor(opts: { dataDir: string; interval: number; logger?: TriggerLogger }) { + this.dataDir = opts.dataDir; + this.interval = opts.interval; + this.logger = opts.logger; + } + + async shouldGenerate(): Promise { + const cpManager = new CheckpointManager(this.dataDir); + const cp = await cpManager.read(); + this.logger?.debug?.(`${TAG} Evaluating: total_processed=${cp.total_processed}, last_persona_at=${cp.last_persona_at}, memories_since=${cp.memories_since_last_persona}, scenes=${cp.scenes_processed}`); + + // Priority 1: Agent explicitly requested persona update + if (cp.request_persona_update) { + const result: TriggerResult = { + should: true, + reason: `主动请求: ${cp.persona_update_reason || "Agent 请求更新"}`, + }; + this.logger?.debug?.(`${TAG} Trigger P1 (explicit request): ${result.reason}`); + return result; + } + + // Priority 2: Cold start — first extraction done, no persona yet, has scene files + if ( + cp.scenes_processed > 0 && + cp.last_persona_at === 0 && + (await this.hasSceneFiles()) + ) { + const result: TriggerResult = { should: true, reason: "首次冷启动：首次提取完成且有场景文件" }; + this.logger?.debug?.(`${TAG} Trigger P2 (cold start): scenes_processed=${cp.scenes_processed}, total_processed=${cp.total_processed}`); + return result; + } + + // Priority 2.5: Recovery — persona was generated before but persona.md body + // is now empty (corrupted/missing). Regenerate to restore. + if ( + cp.last_persona_at > 0 && + (await this.hasSceneFiles()) && + !(await this.hasPersonaBody()) + ) { + const result: TriggerResult = { should: true, reason: "恢复：persona.md 正文丢失或为空，需要重新生成" }; + this.logger?.debug?.(`${TAG} Trigger P2.5 (recovery): last_persona_at=${cp.last_persona_at}, persona body missing`); + return result; + } + + // Priority 3: First scene block extraction + if (cp.scenes_processed === 1 && cp.memories_since_last_persona > 0) { + const result: TriggerResult = { should: true, reason: "首次 Scene Block 提取完成" }; + this.logger?.debug?.(`${TAG} Trigger P3 (first scene): scenes_processed=${cp.scenes_processed}`); + return result; + } + + // Priority 4: Reached threshold + if (cp.memories_since_last_persona >= this.interval) { + const result: TriggerResult = { + should: true, + reason: `达到阈值: ${cp.memories_since_last_persona} >= ${this.interval}`, + }; + this.logger?.debug?.(`${TAG} Trigger P4 (threshold): ${result.reason}`); + return result; + } + + this.logger?.debug?.(`${TAG} No trigger conditions met`); + return { should: false, reason: "" }; + } + + private async hasSceneFiles(): Promise { + const blocksDir = path.join(this.dataDir, "scene_blocks"); + try { + const files = await fs.readdir(blocksDir); + const hasFiles = files.some((f) => f.endsWith(".md")); + return hasFiles; + } catch { + return false; + } + } + + /** + * Check whether persona.md has a non-empty body (excluding scene navigation). + * Returns false if the file doesn't exist, is empty, or only contains + * scene navigation (no actual persona content). + */ + private async hasPersonaBody(): Promise { + const personaPath = path.join(this.dataDir, "persona.md"); + try { + const raw = await fs.readFile(personaPath, "utf-8"); + const body = stripSceneNavigation(raw).trim(); + return body.length > 0; + } catch { + return false; + } + } +} diff --git a/src/prompts/l1-dedup.ts b/src/prompts/l1-dedup.ts new file mode 100644 index 0000000..33db8f0 --- /dev/null +++ b/src/prompts/l1-dedup.ts @@ -0,0 +1,163 @@ +/** + * L1 Conflict Detection Prompt (Batch Mode) + * + * Based on Kenty's validated prototype prompt (l1_conflict_detection_prompt.md). + * Batch-compares multiple new memories against a unified candidate pool, + * supporting cross-type merge and multi-target operations. + */ + +import type { MemoryRecord, ExtractedMemory } from "../record/l1-writer.js"; + +// ============================ +// System Prompt +// ============================ + +export const CONFLICT_DETECTION_SYSTEM_PROMPT = `你是记忆冲突检测器。批量比较多条【新记忆】与【统一候选记忆池】中的已有记忆，逐条决定如何处理。 + +## 核心规则 + +- **跨 type 合并**：不同 type（persona / episodic / instruction）的记忆如果语义上描述同一事实/事件，**可以合并**。 +- **多对多合并**：一条新记忆可以同时替换/合并候选池中的**多条**已有记忆（通过 target_ids 数组指定）。 +- 合并后你必须判断新记忆的最佳 type（merged_type）。 + +## 判断逻辑 + +1. **分辨记忆性质**： + - **状态类**（persona/instruction）：偏好、特质、长期设定、相对稳定的事实、行为规则 + - **事件类**（episodic）：一次性经历、带时间点的客观记录，建议合并同一件事的前因后果 + +2. **判断是否同一事实/事件**：主体相同、主题一致、时间接近、scene_name 相似 + +3. **选择动作**： + - "store"：视为新信息，新增当前记忆。 + - "skip"：已有记忆更好，新记忆无增量或更模糊，忽略当前记忆。 + - "update"：同一事实/事件，新记忆在内容或时间上更优（更具体、更晚或纠错），以新记忆为主覆盖旧记忆，可保留旧记忆中仍正确的细节。 + - "merge"：同一事实或同一演化过程，多条记忆信息互补且不矛盾，合并成一条更完整记忆，信息尽量不冗余。 + +4. **策略倾向**： + - 状态类：多条描述同一偏好/特质 → 倾向 merge；无增量 → skip；明确更新 → update + - 事件类：同一事件的前因后果、不同阶段 → 倾向 merge 为一条完整叙述；完全相同 → skip + - 跨类型示例：一条 episodic "用户在 2018 年开始做播客" + 一条 persona "用户有播客制作经验" → 可 merge 为一条 persona 或 episodic（取决于信息侧重） + +5. **timestamp 处理**： + - merge / update 时，merged_timestamps 应包含**所有相关记忆的时间戳并集**（去重排序） + - 这样可以保留事件发生的完整时间线 + +## 输出格式 + +严格输出 JSON 数组，每个元素对应一条新记忆的决策。不输出任何其他内容： + +[ + { + "record_id": "新记忆的 record_id", + "action": "store|update|skip|merge", + "target_ids": ["要删除的候选记忆 record_id 1", "record_id 2"], + "merged_content": "合并/更新后的记忆内容（merge/update 时必填）", + "merged_type": "合并后的最佳 type：persona|episodic|instruction（merge/update 时必填）", + "merged_priority": 85, + "merged_timestamps": ["合并后的时间戳数组，包含所有新旧记忆时间戳的并集（merge/update 时必填）"] + } +] + +字段说明： +- target_ids：要删除替换的旧记忆 ID **数组**（可以 1 条或多条）。store/skip 时省略或为空。 +- merged_content：merge/update 时的最终记忆文本。store/skip 时省略。 +- merged_type：merge/update 后记忆应归属的 type。根据合并后内容本质判断。 +- merged_priority：merge/update 后的新优先级（0-100 整数，merge/update 时必填）。合并后信息更完整、更确定，通常应**酌情提升** priority（例如两条 priority 70 的记忆合并后可提升到 80）。参考标准：80-100（核心特质/重要事件），60-79（一般偏好/普通活动），<60（次要信息）。 +- merged_timestamps：合并后的时间戳数组。收集新记忆 + 所有被合并旧记忆的时间戳，去重排序。`; + +// ============================ +// Prompt Builder +// ============================ + +/** + * Candidate search result for a single new memory. + */ +export interface CandidateMatch { + newMemory: ExtractedMemory & { record_id: string }; + candidates: MemoryRecord[]; +} + +/** + * Format the batch conflict detection prompt using a unified candidate pool. + * + * Format (aligned with prototype): + * 1. Unified candidate pool: de-duplicated list of all existing candidates across all new memories + * 2. Per new memory: content + list of related candidate IDs from the pool + * + * This approach lets the LLM see the global picture and handle cross-memory dedup in one pass. + * + * @param matches - Array of new memories with their candidate matches + */ +export function formatBatchConflictPrompt(matches: CandidateMatch[]): string { + // Step 1: Build unified candidate pool (de-duplicate across all new memories) + const unifiedPool = new Map(); + const perMemoryCandidateIds = new Map(); + + for (const m of matches) { + const candidateIds: string[] = []; + for (const c of m.candidates) { + if (!unifiedPool.has(c.id)) { + unifiedPool.set(c.id, c); + } + candidateIds.push(c.id); + } + perMemoryCandidateIds.set(m.newMemory.record_id, candidateIds); + } + + // Step 2: Format unified pool as JSON + const poolList = Array.from(unifiedPool.values()).map((c) => ({ + record_id: c.id, + content: c.content, + type: c.type, + priority: c.priority, + scene_name: c.scene_name, + timestamps: c.timestamps, + })); + + let poolSection: string; + if (poolList.length === 0) { + poolSection = "## 统一候选记忆池\n\n（空，没有已有记忆，所有新记忆直接 store）"; + } else { + const poolStr = JSON.stringify(poolList, null, 2); + poolSection = `## 统一候选记忆池（共 ${poolList.length} 条已有记忆）\n\n${poolStr}`; + } + + // Step 3: Format each new memory with its related candidate IDs + const memoryParts = matches.map((m, idx) => { + const relatedIds = perMemoryCandidateIds.get(m.newMemory.record_id) ?? []; + const relatedNote = + relatedIds.length > 0 + ? JSON.stringify(relatedIds) + : "[]（无相似候选，直接 store）"; + + const memStr = JSON.stringify( + { + record_id: m.newMemory.record_id, + content: m.newMemory.content, + type: m.newMemory.type, + priority: m.newMemory.priority, + scene_name: m.newMemory.scene_name, + }, + null, + 2, + ); + + return `### 第 ${idx + 1} 条新记忆 (record_id: ${m.newMemory.record_id})\n${memStr}\n\n【关联候选 ID】${relatedNote}`; + }); + + const newMemoriesText = memoryParts.join( + "\n\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n\n", + ); + + // Step 4: Assemble final prompt + return `${poolSection} + +${"═".repeat(50)} + +## 待判断的新记忆（共 ${matches.length} 条） + +${newMemoriesText} + +请逐条判断并输出决策 JSON 数组。当某条新记忆的候选列表为空时，该条直接输出 action=store。`; +} diff --git a/src/prompts/l1-extraction.ts b/src/prompts/l1-extraction.ts new file mode 100644 index 0000000..6378598 --- /dev/null +++ b/src/prompts/l1-extraction.ts @@ -0,0 +1,137 @@ +/** + * L1 Extraction Prompt: 情境切分 + 记忆提取 + * + * Based on Kenty's validated prototype prompt (l1_memory_extraction_prompt.md). + * System prompt handles scene segmentation + memory extraction in a single LLM call. + * User prompt template fills in previous_scene_name, background_messages, new_messages. + */ + +import type { ConversationMessage } from "../conversation/l0-recorder.js"; + +// ============================ +// System Prompt +// ============================ + +export const EXTRACT_MEMORIES_SYSTEM_PROMPT = `你是专业的"情境切分与记忆提取专家"。 +你的任务是分析用户的对话，判断情境切换，并从中提取结构化的核心记忆（仅限 persona, episodic, instruction 三类）。 + +### 任务一：情境切分（Scene Segmentation） +分析【待提取的新消息】，结合【上一个情境】，判断并输出当前对话的情境。 +- 继承：无明显切换，沿用上一个情境。 +- 切换条件：用户发出明确指令（如"换话题"）、意图转变、或提出独立新目标。 +- 一段对话可能只有一个情境，也可能有多个情境（话题多次切换时）。 +- 命名规则："我（AI）在和xxx（用户身份）做xxx（目标活动）"（中文，30-50字，单句，全局唯一）。 + +--- + +### 任务二：核心记忆提取（Memory Extraction） +结合背景和当前情境，仅从【待提取的新消息】中提取核心信息。 + +【通用提取原则】 +1. 宁缺毋滥：过滤琐碎闲聊、临时性指令和一次性操作（如"这次、本单"）；剔除不可靠的边缘信息。 +2. 独立完整：记忆必须"跳出当前对话依然成立"，无上下文也能看懂。提取主体必须以"用户（姓名）"或"AI"为核心。 +3. 归纳合并：强关联或因果关系的多条消息，必须合并为一条完整记忆，不可碎片化。 + +【支持提取的三大类型】（必须严格遵守类型规则） + +1. 个性化记忆 (type: "persona") + - 定义：用户的稳定属性、偏好、技能、价值观、习惯（如住所、职业、饮食禁忌）。 + - 提取句式："用户（[姓名]）喜欢/是/擅长..." + - 打分 (priority)：80-100（健康/禁忌/核心特质）；50-70（一般喜好/技能）；<50（模糊次要，可丢弃）。 + - 触发词：喜欢、习惯、经常、我这个人... + +2. 客观事件记忆 (type: "episodic") + - 定义：客观发生的动作、决定、计划或达成结果。绝不包含纯主观感受。 + - 提取句式："用户（[姓名]）在 [最好是精确绝对时间] 于 [地点] [做了某事（可以包含起因、经过、结果）]"。 + - 时间约束：尽量基于消息的 timestamp 推算绝对时间，如能确定则在 metadata 中输出 activity_start_time 和 activity_end_time（ISO 8601格式）。无法确定时可省略。 + - 打分 (priority)：80-100（重要事件/计划）；60-70（一般完整活动）；<60（琐碎事项，直接丢弃）。 + +3. 全局指令记忆 (type: "instruction") + - 定义：用户对 AI 提出的长期行为规则、格式偏好、语气控制。 + - 提取句式："用户要求/希望 AI 以后回答时..." + - 触发词：以后都、从现在开始、记住、必须。 + - 打分 (priority)：-1（极其严格的全局死命令）；90-100（核心行为规则）；70-80（重要要求）；<70（临时要求，直接丢弃）。 + +--- + +### 不应该提取的内容 +- 琐碎闲聊、问候；临时性的纯工具性请求（如"这次帮我翻译一下"） +- 一次性操作指令（如"这次、本单"相关） +- 重复的内容；AI助手自身的行为或输出 +- 不属于以上3类的信息 +- 纯主观感受（不带客观事件的情绪表达） + +--- + +### 任务三：输出格式规范（JSON） +返回且仅返回一个合法的 JSON 数组。数组的每一项是一个情境，包含该情境的消息范围和抽取到的记忆： + +[ + { + "scene_name": "当前生成或继承的情境名称", + "message_ids": ["属于该情境的消息ID列表"], + "memories": [ + { + "content": "完整、独立的记忆陈述（按对应类型的句式要求）", + "type": "persona|episodic|instruction", + "priority": 80, + "source_message_ids": ["消息ID_1", "消息ID_2"], + "metadata": {} + } + ] + } +] + +metadata 字段说明： +- episodic 类型：如能确定活动时间，填入 {"activity_start_time": "ISO8601", "activity_end_time": "ISO8601"} +- 其他类型或无法确定时间：输出空对象 {} + +如果整段对话无有意义的记忆，也要输出情境分割结果，memories 为空数组： +[ + { + "scene_name": "情境名称", + "message_ids": ["id1", "id2"], + "memories": [] + } +] + +请严格按上述 JSON 数组格式输出，不要输出任何额外的 Markdown 代码块修饰符（如 \`\`\`json）或解释文本。`; + +// ============================ +// Prompt Builder +// ============================ + +/** + * Format the user prompt for L1 extraction. + * + * @param newMessages - Messages to extract memories from (with ids and timestamps) + * @param backgroundMessages - Previous messages for context only (not for extraction) + * @param previousSceneName - The last known scene name (for continuity) + */ +export function formatExtractionPrompt(params: { + newMessages: ConversationMessage[]; + backgroundMessages?: ConversationMessage[]; + previousSceneName?: string; +}): string { + const { newMessages, backgroundMessages = [], previousSceneName = "无" } = params; + + const bgText = backgroundMessages.length > 0 + ? backgroundMessages + .map((m) => `[${m.id}] [${m.role}] [${new Date(m.timestamp).toISOString()}]: ${m.content}`) + .join("\n\n") + : "无"; + + const newText = newMessages + .map((m) => `[${m.id}] [${m.role}] [${new Date(m.timestamp).toISOString()}]: ${m.content}`) + .join("\n\n"); + + return `【上一个情境】：${previousSceneName} + +【背景对话】（仅供理解上下文推断关系/时间，严禁从中提取记忆）： +${bgText} + +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +【待提取的新消息】（务必结合 timestamp 推算时间，只从这里提取记忆！）： +${newText}`; +} diff --git a/src/prompts/persona-generation.ts b/src/prompts/persona-generation.ts new file mode 100644 index 0000000..4a42b01 --- /dev/null +++ b/src/prompts/persona-generation.ts @@ -0,0 +1,172 @@ +/** + * Persona Generation Prompt — instructs LLM to generate/update user persona + * using the four-layer deep scan model. + * + * v2: Updated prompt with anti-hallucination guardrails, richer output + * template with per-chapter writing guidance, and streamlined iteration guide. + */ + +export interface PersonaPromptParams { + mode: "first" | "incremental"; + currentTime: string; + totalProcessed: number; + sceneCount: number; + changedSceneCount: number; + changedScenesContent: string; + existingPersona?: string; + triggerInfo?: string; + /** @deprecated Kept for call-site compatibility; no longer used in prompt. */ + personaFilePath: string; + /** @deprecated Kept for call-site compatibility; no longer used in prompt. */ + checkpointPath: string; +} + +export function buildPersonaPrompt(params: PersonaPromptParams): string { + const { + mode, + currentTime, + totalProcessed, + sceneCount, + changedSceneCount, + changedScenesContent, + existingPersona, + triggerInfo, + } = params; + + const modeLabel = mode === "first" ? "🆕 首次生成" : "🔄 迭代更新"; + + const triggerSection = triggerInfo + ? `\n### 触发信息\n${triggerInfo}\n` + : ""; + + // Existing persona is now placed AFTER scene blocks (closer to the + // processing instructions) so the LLM sees fresh evidence first. + const existingPersonaSection = existingPersona + ? `\n## 📄 当前 Persona（工程已预加载）\n\n` + + `*以下是现有 persona.md 的完整内容（${existingPersona.length} 字符），基于此更新后请控制在2000字内：*\n\n` + + `\`\`\`markdown\n${existingPersona}\n\`\`\`\n\n---\n` + : ""; + + const iterationGuide = mode === "incremental" + ? `\n## 🔄 迭代决策指南\n\n` + + `面对变化场景，自主判断处理方式：强化（佐证已有洞察）/ 补充（新维度）/ 修正（矛盾）/ 重构（结构调整）/ 不改（无有用新增内容）。\n` + : ""; + + return `# 🧬 Persona Architect - Incremental Evolution Protocol + +**⏰ 更新时间**: ${currentTime} +**模式**: ${modeLabel} + +请你结合已有的 persona.md 和新增/变化的 block 信息深度分析，然后使用文件工具将结果写入 \`persona.md\` 文件。 + +## ⛔ 文件操作约束（必须严格遵守） + +1. **必须使用文件工具将最终 persona 内容写入 \`persona.md\`**。当前工作目录已设为数据目录，直接使用文件名 \`persona.md\`。 + - **首次生成 / 大幅重写**：使用 \`write_to_file('persona.md', 完整内容)\` 整体写入 + - **增量更新（局部修改）**：使用 \`replace_in_file('persona.md', 旧内容片段, 新内容片段)\` 精确替换，可节省开销 +2. **只能操作 \`persona.md\` 这一个文件**，禁止读取或写入任何其他文件（包括 scene_blocks/、.metadata/ 等）。 +3. **写入的内容必须只包含最终的 persona 文档**，不要包含你的思考过程、分析步骤或任何非 persona 内容。 +4. **无需 read_file**：当前 persona.md 的完整内容已在下方提供，直接基于它进行更新即可。 + +### 🚫 严格禁止 +- **禁止过长**：persona.md 内容总长度不要超过 2000 字符，及时做总结和删除不重要的信息。 +- **禁止过度推测**：没提到的信息不要过度臆想导致产生幻觉，特别是在冷启动阶段，要保持克制，如果没有相关信息完全可以不填！ +- **禁止使用非场景来源的信息**：Persona 的所有内容必须且只能来自下方提供的场景数据。不要从 workspace 目录结构、文件路径、系统信息等技术元数据中提取任何关于用户的个人信息。 +- **禁止操作 persona.md 以外的任何文件**。 + +--- +${triggerSection} +## 📊 统计 +- **总记忆数**: ${totalProcessed} 条 +- **场景总数**: ${sceneCount} 个 +- **变化场景**: ${changedSceneCount} 个（自上次更新后） + +--- +${changedScenesContent} + + +${existingPersonaSection} +## ⚙️ 核心运作逻辑 (The Core Logic) + +🧠 核心思维引擎：连接与综合 (Connect & Synthesize) +请遵循 "叙事连贯性" 原则处理信息。禁止简单的罗列（No Bullet-point Spamming）。 + +1. 寻找"贯穿线" (The Connecting Thread) +不要孤立地看信息。要寻找不同领域行为背后的共同逻辑。 +** 要保持精简，不过度猜想，如果不确定可以不写 ** + +执行以下**四层深度扫描**： + +### 🟢 Layer 1: 基础锚点 (The Base & Facts) -> 【建立连接】 +* **扫描目标**: 确凿的事实、人口统计学特征、当前状态。 +* **实用价值**: 为 Agent 提供**破冰话题**和**上下文感知**。 + +### 🔵 Layer 2: 兴趣图谱 (The Interest Graph) -> 【提供谈资】 +* **扫描目标**: 用户投入时间、金钱或注意力的事物。 +* **提取原则**: **区分活跃度**（活跃爱好 / 被动消费 / 休眠兴趣）。 +* **实用价值**: 让 Agent 能够进行**高质量的闲聊 (Chit-chat)** 和 **生活推荐**。 + +### 🟡 Layer 3: 交互协议 (The Interface) -> 【消除摩擦】 +* **扫描目标**: 用户的沟通习惯、雷区、工作流偏好。 +* **实用价值**: 指导 Agent **如何说话、如何交付结果**，避免踩雷。 + +### 🔴 Layer 4: 认知内核 (The Core) -> 【深度共鸣】 +* **扫描目标**: 决策逻辑、矛盾点、终极驱动力。 +* **实用价值**: 让 Agent 成为**能够替用户做决策**的"副驾驶"。 +${iterationGuide} +--- + +## 📝 输出模板 (The Persona Template) + +请参考以下格式，使用 \`write_to_file('persona.md', ...)\` 写入最终内容。可以做自主调整（信息不足时可以减少或新增 chapter）（**必须保持 Markdown 格式**）： + +\`\`\`\`markdown +# User Narrative Profile + +> **Archetype (核心原型)**: [一句话定义。例如：一位在现实重力下挣扎，但试图通过技术构建理想国的"务实理想主义者"。] + +> **基本信息** +（用户的基本信息，如年龄、性别、职业等，更新时若有冲突则覆盖，不冲突尽量叠加） + - + - + +> **长期偏好** +（你观察到的用户最稳定且可复用的偏好） + - + - + +## 📖 Chapter 1: Context & Current State (全景语境) +*(将基础事实与当前状态融合，写成一段连贯的背景介绍)* + +**[这里写连贯描述，区别较大的时候可以分点阐述]** + +## 🎨 Chapter 2: The Texture of Life (生活的肌理) +*(将兴趣、消费、生活习惯串联起来，展示生活品味)* + +**[这里写连贯的描述，重点在于"兴趣/偏好"和"品味"的统一性，区别较大的时候可以分点阐述]** + +## 🤖 Chapter 3: Interaction & Cognitive Protocol (交互与认知协议) +*(这是 Main Agent 的行动指南。为了实用，这里保持半结构化，但要解释"为什么")* + +### 3.1 沟通策略 (How to Speak) +### 3.2 决策逻辑 (How to Think) + +## 🧩 Chapter 4: Deep Insights & Evolution (深层洞察与演变) +*(人类学观察笔记)* + +* **矛盾统一性**: [描述用户身上看似冲突但实则合理的特质]。 +* **演变轨迹**: [可加上时间，分为多点，描述用户最近发生的变化]。 +* **涌现特征**: 提炼 3-7 个最核心的特质标签，每个标签单独一行并附上简短注释（10-15字） + - \`TagName\` - 简短注释说明 +\`\`\`\` + +--- + +### ⚠️ 成功标准 +- ✅ **必须使用 \`write_to_file\` 或 \`replace_in_file\` 工具写入最终结果到 \`persona.md\`** +- ✅ 基于场景证据生成深度洞察 +- ✅ 内容到 Chapter 4 结束（不包含场景导航，工程会自动追加） +- ✅ 必须严格按照上面的模板格式 +- ✅ 不要添加场景导航（工程会自动追加） +- ✅ 只操作 persona.md，不要操作其他文件`; +} diff --git a/src/prompts/scene-extraction.ts b/src/prompts/scene-extraction.ts new file mode 100644 index 0000000..19e8f06 --- /dev/null +++ b/src/prompts/scene-extraction.ts @@ -0,0 +1,228 @@ +/** + * Scene Extraction Prompt — instructs LLM to consolidate memories into scene blocks + * using file tools (read_file, write_to_file, replace_in_file). + * + * Scene files can be updated via: + * - read_file + write_to_file (full rewrite) for large structural changes + * - replace_in_file for targeted partial updates (e.g. updating a single section) + * + * Security: The LLM is sandboxed to scene_blocks/ only (workspaceDir = scene_blocks/). + * It has NO visibility into checkpoint, scene_index, persona.md, or any other system file. + * File deletion is achieved via "soft-delete" — writing an empty string to the file + * — and the SceneExtractor subsequently removes empty files with fs.unlink. + * + * Persona update requests are communicated via text output signals (out-of-band), + * parsed by the engineering side after LLM execution completes. + */ + +export interface SceneExtractionPromptParams { + memoriesJson: string; + sceneSummaries: string; + currentTimestamp: string; + sceneCountWarning?: string; + /** List of existing scene filenames (relative, e.g. ["work.md", "hobby.md"]) */ + existingSceneFiles?: string[]; +} + +export function buildSceneExtractionPrompt(params: SceneExtractionPromptParams): string { + const { + memoriesJson, + sceneSummaries, + currentTimestamp, + sceneCountWarning, + existingSceneFiles, + } = params; + + const warningSection = sceneCountWarning + ? `\n⚠️ **场景数量警告**: ${sceneCountWarning}\n` + : ""; + + return `# System Prompt: Memory Consolidation Architect + +## 角色定义 (Role Definition) +你是记忆整合架构师。你的目标是为用户构建一个"数字第二大脑"。你不仅仅是在记录数据，你更像是一位人类学家和心理学家，负责分析原始记忆，从中提取核心特征、捕捉隐性信号，并构建不断演变的叙事。 + + +## 架构模型 + +### Layer 1 (Input): Raw Memories +- **来源**：API 分批召回（每批 20 条） +- **状态**：碎片化、无序 + +### Layer 2 (Processing): Scene Diaries +- **形态**：**不是清单，是连贯的叙事文档** +- **逻辑**：将 L1 碎片融合进特定场景文件（强制限制在15个以内） +- **动作**：Create（创建）、Integrate（整合）、Rewrite（重写） +- **禁止**：简单追加列表 + +你主要负责L1到L2的生成任务 +${warningSection} +## 输入环境 (Input Context) +你将接收三个输入： +1. 新增记忆 (New Memory): 一段原始的、非结构化的新近回忆信息。 +2. 现有 Block 映射表 (Existing Blocks Map): 包含当前所有记忆块（Markdown 文件）的文件名和摘要的列表。 +3. 当前时间 (Current Time): 用于生成元数据的具体时间戳。 + + +### 1️⃣ New Memories List +${memoriesJson} + +### 2️⃣ Existing Scene Blocks Summary +${sceneSummaries} + +### 3️⃣ Current Timestamp +${currentTimestamp} + +${existingSceneFiles && existingSceneFiles.length > 0 + ? `### 📁 已有场景文件清单（仅以下文件可 read_file） +${existingSceneFiles.map((f) => `- \`${f}\``).join("\n")} +` + : `### 📁 已有场景文件清单 +（当前无已有场景文件） +`} +## ⛔ 文件操作约束（必须严格遵守） +1. **所有文件操作使用相对文件名**（如 \`技术研究-Rust学习.md\`），当前工作目录已设为场景文件目录 +2. **read_file 只能读取上面"已有场景文件清单"中列出的文件**，禁止猜测或编造不在清单中的文件名 +3. **创建新场景文件时**，直接使用文件名，如 \`新场景名.md\` +4. **场景文件支持 replace_in_file**。对于局部更新（如只更新某个章节或 META 字段），可以使用 \`replace_in_file\` 进行精确替换。对于大范围重写或结构性变更，建议使用 \`read_file\` + \`write_to_file\` 整体重写。 +5. **场景索引和系统配置由工程系统自动维护**，你只需专注于操作 \`.md\` 场景文件 + +## 工作流与逻辑 (Workflow & Logic) +在生成输出之前，你必须执行以下"思维链"过程： + +### ⚠️ 阶段 0：强制检查场景总数（必须先执行） + +**在处理任何记忆之前，你必须：** + +1. **统计当前场景总数**：检查 "Existing Scene Blocks Summary" 中的场景数量 +2. **遵守分级预警，上限为15个block**： + - 红色预警（≥ 15）：**必须先合并**，将最相似的 2-4 个场景合并为 1 个，然后再处理新记忆 + - 橙色预警（= 15-1）：**只能 UPDATE 现有场景，不能 CREATE 新场景** + - 黄色预警（接近15）：**优先 UPDATE 或主动 MERGE 相似场景** + +**合并优先级**（当需要合并时，按以下顺序选择）： +1. **主题高度重叠**：如"Python后端开发"和"Go后端开发" → 合并为"后端开发技术栈" +2. **叙事弧线相同**：如"求职材料-JD匹配"和"职业发展-能力对齐" → 合并为"职业发展与求职" +3. **热度最低的场景**：如果没有明显重叠，合并或删除 heat 最低的 2-3 个场景 + +### 阶段 1：分析与分类 +分析 新增记忆。它的核心领域是什么？（例如：编程风格、情绪状态、职业轨迹、人际关系）。 +提取事实事件链（触发 -> 行动 -> 结果）以及底层的心理状态。 + +### 阶段 2：检索与策略选择 +将新记忆与 现有 Block 映射表 进行比对。 +需要时使用 \`read_file\` 工具读取完整场景文件内容 +**只能读取上面"已有场景文件清单"中列出的文件，禁止猜测其他文件路径。** + +**核心原则：默认策略是 UPDATE，不是 CREATE。** 当犹豫于 UPDATE 和 CREATE 之间时，选择 UPDATE。 + +策略选择（按优先级排序）： +1. **UPDATE（更新）**【首选策略】: 如果存在相关的 Block（基于摘要或文件名的相似性），先 read 文件内的具体信息，再锁定该 Block 进行更新（write 或 replace） +2. **MERGE（合并）**: + - 合并的新 block 应该是生成概括性更强的场景，包含已有的多个相似场景 + - **强制合并**：当前 Block 总数 **≥ 上限**时，必须先将多个相似记忆合并，或者删除最旧或者最不重要的 block + - **主动合并**：即使未达上限，如果两个 Block 属于同一叙事弧线，也应合并以增加深度 +3. **CREATE（新建）**【最后手段】: + - **前提条件**：当前场景总数未达上限 + - **CREATE 前的强制验证**：必须先用 \`read_file\` 检查至少 2 个最相似的现有场景，确认新记忆确实无法融入后才能 CREATE。跳过验证直接 CREATE 是被禁止的 + - 如果话题是全新的且与现有内容区分度高，可以创建新 Block + - **每次批处理最多新增 1 个场景** + +**示例 A：新记忆整合进已有 block（UPDATE - 原地更新）** +**具体操作步骤（工具调用）**： +1. \`read_file('Python后端开发.md')\` → 获取已有内容 A +2. 分析新记忆 + 已有内容 A → 整合生成新内容 B（\`heat = 旧heat + 1\`） +3. \`write_to_file('Python后端开发.md', B)\` → **整体重写该场景文件** + 或 \`replace_in_file('Python后端开发.md', old_section, new_section)\` → **局部更新某部分** + +合并多个 block 的逻辑： +**具体操作步骤（工具调用）**： +1. \`read_file('Python后端开发.md')\` → 获取内容 A +2. \`read_file('Go后端开发.md')\` → 获取内容 B +3. 整合 A + B + 新记忆 → 生成新内容 C（\`heat = heatA + heatB + 1\`） +4. \`write_to_file('后端开发技术栈.md', C)\` → 创建新文件，写入合并后的完整内容 +5. \`write_to_file('Python后端开发.md', '')\` → **清空旧文件 A（标记删除）** +6. \`write_to_file('Go后端开发.md', '')\` → **清空旧文件 B（标记删除）** + +### 阶段 3：撰写与合成（核心任务） +深度整合: 严禁简单的文本追加。你必须结合上下文（基于摘要或提供的原始内容）重写叙事，将新信息自然地融入其中。 +隐性推断: 寻找用户 没说出口 的信息。更新"隐性信号"部分。 +冲突检测: 如果新记忆与旧记忆相矛盾，将其记录在"演变轨迹"或"待确认/矛盾点"中。 + +### 撰写准则 (严格遵守) +核心部分禁止列表: "用户核心特征"和"核心叙事"必须是连贯的段落，信息要连贯，可以分段。 +叙事弧线: "核心叙事"必须遵循故事结构（情境 -> 行动 -> 结果）。 + +### 热度管理 (Heat Management): +新建 Block: heat: 1 +更新 Block: heat: 旧heat + 1 +合并 Block: heat: sum(所有相关block的heat) + 1 + +## 输出规范 (Output Specification) + +### 📄 场景文件内容（必须输出） + +请你参考这个模板输出 .md 文件的内容或基于已有md进行更新，每个md控制在1500字符内。不要把模板本身放在 Markdown 代码块中，只需直接输出要写入文件的原始文本。 + +\`\`\`markdown +-----META-START----- +created: {{EXISTING_CREATED_TIME_OR_CURRENT_TIME}} +updated: {{CURRENT_TIME}} +summary: [30-40 words concise summary for indexing] +heat: [Integer] +-----META-END----- + +## 用户基础信息 +[可为空，如果没有可不写这节，可按照需求添加更多点，合并和更新方式尽量叠加，有冲突则覆盖] + -姓名： + -职业： + -居住地： + - …… + +## 用户核心特征 +[这里不是列表！是一段连贯的描述。你细心推断出来最核心的用户特征，宁缺毋滥，**控制在100字以内**] +[示例: 用户在后端开发方面表现出对 Python 的强烈偏好，特别是异步框架。近期（2026-02）开始关注 Rust 的所有权机制，这表明用户有向系统级编程转型的意图。] + +## 用户偏好 +[这里可以是列表！**如果没有可以为不写这节**，记录用户明确的偏好信息（显性偏好），注意不要重复信息，不要流水账，偏好要可复用，更新时可以动态整合甚至重写] +[示例：用户喜欢吃苹果] + +## 隐性信号 +[这是给人类学家看的，记录那些"没明说但很重要"的事，和显性偏好不一样，一定是你推断出来的，需要深思熟虑后再生成，可以为空，宁缺毋滥。你可以随时更新/删除/修改这里的信息] + +## 核心叙事 +[这里不是列表！是一段连贯的描述，**控制在400字以内**，注意不要重复信息，不要流水账，可以动态整合甚至重写] +*(这里记录连贯的故事，必须包含 Trigger -> Action -> Result)* + +[ 示例：本周用户主要集中在后端重构上。初期因为旧代码的耦合度高感到沮丧（**情绪点**），但他拒绝了"打补丁"的建议，坚持进行彻底解耦（**决策点**）。他在此过程中频繁查阅架构设计模式，表现出对"代码洁癖"的执着。] + + +## 演变轨迹 +> [注意] 可以为空，仅记录【用户偏好/性格/重大观念】转变，不记录琐碎、日常更新。当发生冲突时，不要直接覆盖，要记录变化轨迹。 +- [2026-01-10]: 从 "反对加班" 转向 "接受弹性工作"，原因：创业压力（记忆ID: #987） + + +## 待确认/矛盾点 +- [记录当前无法整合的矛盾信息，等待未来记忆澄清] + +\`\`\` + + + +#### 主动触发 Persona 更新（可选） + +**触发条件**：重大价值观转变、跨场景突破性洞察。 + +**触发方式**：在你的 text output 中输出以下标记（不是文件操作）： + +[PERSONA_UPDATE_REQUEST] +reason: 具体原因描述 +[/PERSONA_UPDATE_REQUEST] + + +**执行文件操作**（必须使用工具）： + - 使用 \`read_file\` 读取需要更新的场景文件 + - 使用 \`write_to_file\` 创建新文件或**整体重写**已有场景文件 + - 使用 \`replace_in_file\` 对场景文件进行**局部更新**（如只更新某个章节） + - **删除文件**：使用 \`write_to_file(filename, '')\` 将文件内容清空（系统会自动清理空文件，禁止使用其他删除方式）`; +} diff --git a/src/record/l1-dedup.ts b/src/record/l1-dedup.ts new file mode 100644 index 0000000..a130c91 --- /dev/null +++ b/src/record/l1-dedup.ts @@ -0,0 +1,382 @@ +/** + * L1 Memory Conflict Detection (Batch Mode): decides how to handle multiple new + * memories against existing records in a single LLM call. + * + * v4: Removed JSONL-based Jaccard fallback. Candidate recall now relies exclusively + * on vector search (primary) and FTS5 BM25 (degraded). If neither is available, + * conflict detection is skipped entirely — all memories go straight to store. + * + * Two-phase approach: + * 1. Candidate search per new memory — vector recall or FTS5 keyword recall (fast, no LLM) + * 2. Batch LLM judgment on all new memories + their candidate pools (single call) + */ + +import type { ExtractedMemory, MemoryRecord, DedupDecision, MemoryType } from "./l1-writer.js"; +import { CONFLICT_DETECTION_SYSTEM_PROMPT, formatBatchConflictPrompt } from "../prompts/l1-dedup.js"; +import type { CandidateMatch } from "../prompts/l1-dedup.js"; +import { CleanContextRunner } from "../utils/clean-context-runner.js"; +import { sanitizeJsonForParse } from "../utils/sanitize.js"; +import type { VectorStore } from "../store/vector-store.js"; +import { buildFtsQuery } from "../store/vector-store.js"; +import type { EmbeddingService } from "../store/embedding.js"; + +interface Logger { + debug?: (message: string) => void; + info: (message: string) => void; + warn: (message: string) => void; + error: (message: string) => void; +} + +const TAG = "[memory-tdai][l1-dedup]"; + +// ============================ +// Core function (batch mode) +// ============================ + +/** + * Batch conflict detection: compare all new memories against existing records + * in a single LLM call. + * + * Candidate recall strategy (3-tier degradation): + * 1. Vector recall (vectorStore + embeddingService) — cosine similarity (best) + * 2. FTS5 keyword recall (vectorStore with FTS available) — BM25 ranking (degraded) + * 3. Skip conflict detection entirely — all memories go straight to "store" + * + * The old JSONL-based Jaccard fallback has been removed. If neither vector search + * nor FTS is available, we skip dedup rather than paying the O(N) full-file-scan cost. + * + * @param memories - Newly extracted memories (with record_id) + * @param config - OpenClaw config (for LLM access) + * @param logger - Optional logger + * @param model - Optional model override + * @param vectorStore - Optional vector store for cosine similarity search + * @param embeddingService - Optional embedding service for computing query vectors + * @param conflictRecallTopK - Top-K candidates to recall per new memory (default: 5) + * @returns Array of dedup decisions, one per new memory + */ +export async function batchDedup(params: { + memories: Array; + config: unknown; + logger?: Logger; + model?: string; + /** Vector store for cosine similarity candidate recall */ + vectorStore?: VectorStore; + /** Embedding service for computing query vectors */ + embeddingService?: EmbeddingService; + /** Top-K candidates per new memory (default: 5) */ + conflictRecallTopK?: number; +}): Promise { + const { memories, config, logger, model, vectorStore, embeddingService } = params; + const topK = params.conflictRecallTopK ?? 5; + + if (memories.length === 0) { + return []; + } + + const storeAll = () => + memories.map((m) => ({ + record_id: m.record_id, + action: "store" as const, + target_ids: [], + })); + + // Determine what recall capabilities are available + const hasVectorData = vectorStore && vectorStore.count() > 0; + const hasFts = vectorStore?.isFtsAvailable() ?? false; + + // Fast path: no recall capability at all → skip dedup + if (!hasVectorData && !hasFts) { + logger?.debug?.(`${TAG} No vector data and no FTS available, skipping conflict detection for ${memories.length} memories`); + return storeAll(); + } + + // Phase 1: Find candidates + // + // Decision tree (after the fast-path guard above, vectorStore is guaranteed non-null): + // hasVectorData + embeddingService → Tier 1 vector recall (FTS fallback on error) + // otherwise hasFts → Tier 2 FTS keyword recall + // otherwise → skip dedup (defensive; shouldn't reach here) + let matches: CandidateMatch[]; + + if (hasVectorData && embeddingService) { + // === Tier 1: Vector recall mode === + logger?.debug?.(`${TAG} Using vector recall mode (topK=${topK})`); + try { + matches = await findCandidatesByVector(memories, vectorStore!, embeddingService, topK, logger); + } catch (err) { + logger?.warn?.( + `${TAG} Vector recall failed, falling back to FTS keyword: ${err instanceof Error ? err.message : String(err)}`, + ); + // Degrade to FTS keyword recall + if (hasFts) { + matches = findCandidatesByFts(memories, vectorStore!, logger); + } else { + logger?.debug?.(`${TAG} FTS not available either, skipping conflict detection`); + return storeAll(); + } + } + } else if (hasFts) { + // === Tier 2: FTS keyword recall === + logger?.debug?.(`${TAG} Using FTS keyword recall mode (no embedding service or no vector data)`); + matches = findCandidatesByFts(memories, vectorStore!, logger); + } else { + // Shouldn't reach here given the fast-path check above, but be defensive + logger?.debug?.(`${TAG} No usable recall path, skipping conflict detection`); + return storeAll(); + } + + // Check if any memory has candidates + const hasAnyCandidates = matches.some((m) => m.candidates.length > 0); + + if (!hasAnyCandidates) { + logger?.debug?.(`${TAG} No similar records found for any memory, all will be stored`); + return storeAll(); + } + + // Phase 2: Batch LLM judgment + return runLlmJudgment(matches, memories, config, logger, model); +} + +/** + * Phase 2: Run batch LLM judgment on candidate matches. + */ +async function runLlmJudgment( + matches: CandidateMatch[], + memories: Array, + config: unknown, + logger: Logger | undefined, + model: string | undefined, +): Promise { + logger?.debug?.(`${TAG} Running batch conflict detection for ${memories.length} memories`); + + try { + const runner = new CleanContextRunner({ + config, + modelRef: model, + enableTools: false, + logger, + }); + + const userPrompt = formatBatchConflictPrompt(matches); + + const result = await runner.run({ + prompt: userPrompt, + systemPrompt: CONFLICT_DETECTION_SYSTEM_PROMPT, + taskId: "l1-conflict-detection", + timeoutMs: 180_000, + // maxTokens: 4000, remove maxTokens use model default or inherit from config + }); + + const decisions = parseBatchResult(result, memories, logger); + return decisions; + } catch (err) { + logger?.warn?.( + `${TAG} Batch conflict detection failed, defaulting all to store: ${err instanceof Error ? err.message : String(err)}`, + ); + return memories.map((m) => ({ + record_id: m.record_id, + action: "store" as const, + target_ids: [], + })); + } +} + +// ============================ +// Candidate recall strategies +// ============================ + +/** + * Vector-based candidate recall (aligned with prototype): + * batch-embed new memories → cosine search in VectorStore → exclude self-batch → return candidates. + */ +async function findCandidatesByVector( + memories: Array, + vectorStore: VectorStore, + embeddingService: EmbeddingService, + topK: number, + logger?: Logger, +): Promise { + const newRecordIds = new Set(memories.map((m) => m.record_id)); + + // Batch-compute embeddings for all new memories + const texts = memories.map((m) => m.content); + const embeddings = await embeddingService.embedBatch(texts); + + const matches: CandidateMatch[] = []; + + for (let i = 0; i < memories.length; i++) { + const mem = memories[i]; + const queryVec = embeddings[i]; + + // Vector search top-K (request extra to account for self-batch filtering) + const searchResults = vectorStore.search(queryVec, topK + memories.length); + + // Exclude records from current batch, convert to MemoryRecord format + const candidates: MemoryRecord[] = searchResults + .filter((r) => !newRecordIds.has(r.record_id)) + .slice(0, topK) + .map((r) => ({ + id: r.record_id, + content: r.content, + type: r.type as MemoryRecord["type"], + priority: r.priority, + scene_name: r.scene_name, + source_message_ids: [], + metadata: {}, + timestamps: [r.timestamp_str].filter(Boolean), + createdAt: "", + updatedAt: "", + sessionKey: r.session_key, + sessionId: r.session_id, + })); + + matches.push({ newMemory: mem, candidates }); + } + + logger?.debug?.( + `${TAG} Vector recall: ${matches.map((m) => `${m.newMemory.record_id}→${m.candidates.length}`).join(", ")}`, + ); + + return matches; +} + +/** + * FTS5-based candidate recall: + * Uses the FTS index for efficient BM25-ranked keyword matching. + * This replaces the old Jaccard word-overlap fallback entirely. + */ +function findCandidatesByFts( + memories: Array, + vectorStore: VectorStore, + _logger?: Logger, +): CandidateMatch[] { + const newRecordIds = new Set(memories.map((m) => m.record_id)); + const matches: CandidateMatch[] = []; + + for (const mem of memories) { + const ftsQuery = buildFtsQuery(mem.content); + if (ftsQuery) { + const ftsResults = vectorStore.ftsSearchL1(ftsQuery, 10); + // Filter out records from the current batch + const candidates: MemoryRecord[] = ftsResults + .filter((r) => !newRecordIds.has(r.record_id)) + .slice(0, 5) + .map((r) => ({ + id: r.record_id, + content: r.content, + type: r.type as MemoryRecord["type"], + priority: r.priority, + scene_name: r.scene_name, + source_message_ids: [], + metadata: r.metadata_json ? (() => { try { return JSON.parse(r.metadata_json); } catch { return {}; } })() : {}, + timestamps: [r.timestamp_str].filter(Boolean), + createdAt: "", + updatedAt: "", + sessionKey: r.session_key, + sessionId: r.session_id, + })); + matches.push({ newMemory: mem, candidates }); + } else { + matches.push({ newMemory: mem, candidates: [] }); + } + } + + _logger?.debug?.(`${TAG} FTS keyword recall: ${matches.map((m) => `${m.newMemory.record_id}→${m.candidates.length}`).join(", ")}`); + return matches; +} + +// ============================ +// Result parsing +// ============================ + +const VALID_TYPES: MemoryType[] = ["persona", "episodic", "instruction"]; + +/** + * Parse the LLM's batch conflict detection JSON response. + * + * Expected format: [{record_id, action, target_ids, merged_content, merged_type, merged_priority, merged_timestamps}] + */ +function parseBatchResult( + raw: string, + memories: Array, + logger?: Logger, +): DedupDecision[] { + try { + // Strip markdown code block wrappers + let cleaned = raw.trim(); + if (cleaned.startsWith("```")) { + cleaned = cleaned.replace(/^```(?:json)?\s*\n?/, "").replace(/\n?```\s*$/, ""); + } + + // Extract JSON array + const arrayMatch = cleaned.match(/\[[\s\S]*\]/); + if (!arrayMatch) { + logger?.warn?.(`${TAG} No JSON array found in conflict detection response`); + return fallbackStoreAll(memories); + } + + // Sanitize control characters inside JSON string literals that LLM may produce + const sanitized = sanitizeJsonForParse(arrayMatch[0]); + const parsed = JSON.parse(sanitized) as unknown[]; + + if (!Array.isArray(parsed)) { + logger?.warn?.(`${TAG} Conflict detection response is not an array`); + return fallbackStoreAll(memories); + } + + // Build decisions from LLM output + const decisions: DedupDecision[] = []; + const validActions = ["store", "update", "merge", "skip"]; + + for (const item of parsed) { + if (!item || typeof item !== "object") continue; + const d = item as Record; + + const recordId = String(d.record_id ?? ""); + const action = String(d.action ?? "store"); + + if (!validActions.includes(action)) { + logger?.warn?.(`${TAG} Invalid action "${action}" for record ${recordId}, defaulting to store`); + } + + decisions.push({ + record_id: recordId, + action: validActions.includes(action) ? (action as DedupDecision["action"]) : "store", + target_ids: Array.isArray(d.target_ids) ? d.target_ids.map(String) : [], + merged_content: typeof d.merged_content === "string" ? d.merged_content : undefined, + merged_type: VALID_TYPES.includes(d.merged_type as MemoryType) ? (d.merged_type as MemoryType) : undefined, + merged_priority: typeof d.merged_priority === "number" ? d.merged_priority : undefined, + merged_timestamps: Array.isArray(d.merged_timestamps) ? d.merged_timestamps.map(String) : undefined, + }); + } + + // Ensure all memories have a decision (fill missing with "store") + const decidedIds = new Set(decisions.map((d) => d.record_id)); + for (const mem of memories) { + if (!decidedIds.has(mem.record_id)) { + logger?.debug?.(`${TAG} No decision for record ${mem.record_id}, defaulting to store`); + decisions.push({ + record_id: mem.record_id, + action: "store", + target_ids: [], + }); + } + } + + return decisions; + } catch (err) { + logger?.warn?.(`${TAG} Failed to parse conflict detection result: ${err instanceof Error ? err.message : String(err)}`); + return fallbackStoreAll(memories); + } +} + +/** + * Fallback: store all memories when parsing fails. + */ +function fallbackStoreAll(memories: Array): DedupDecision[] { + return memories.map((m) => ({ + record_id: m.record_id, + action: "store" as const, + target_ids: [], + })); +} diff --git a/src/record/l1-extractor.ts b/src/record/l1-extractor.ts new file mode 100644 index 0000000..58f18de --- /dev/null +++ b/src/record/l1-extractor.ts @@ -0,0 +1,500 @@ +/** + * L1 Memory Extractor: extracts structured memories from L0 conversation messages + * using a single LLM call with JSON-mode structured output. + * + * v3: Aligned with Kenty's prompt — scene segmentation + memory extraction in one call, + * followed by batch conflict detection. + * + * Pipeline: + * 1. Read recent messages from L0 (split into background + new) + * 2. Call LLM to extract scene-segmented memories + * 3. Batch conflict detection against existing records + * 4. Write to L1 JSONL files + */ + +import type { ConversationMessage } from "../conversation/l0-recorder.js"; +import { EXTRACT_MEMORIES_SYSTEM_PROMPT, formatExtractionPrompt } from "../prompts/l1-extraction.js"; +import { batchDedup } from "./l1-dedup.js"; +import { writeMemory, generateMemoryId } from "./l1-writer.js"; +import type { ExtractedMemory, MemoryRecord, MemoryType, DedupDecision } from "./l1-writer.js"; +import { CleanContextRunner } from "../utils/clean-context-runner.js"; +import { sanitizeJsonForParse, shouldExtractL1 } from "../utils/sanitize.js"; +import type { VectorStore } from "../store/vector-store.js"; +import type { EmbeddingService } from "../store/embedding.js"; +import { report } from "../report/reporter.js"; + +interface Logger { + debug?: (message: string) => void; + info: (message: string) => void; + warn: (message: string) => void; + error: (message: string) => void; +} + +const TAG = "[memory-tdai][l1-extractor]"; + +// ============================ +// Types +// ============================ + +/** A scene segment with its extracted memories (LLM output) */ +interface SceneSegment { + scene_name: string; + message_ids: string[]; + memories: Array<{ + content: string; + type: string; + priority: number; + source_message_ids: string[]; + metadata: Record; + }>; +} + +export interface L1ExtractionResult { + /** Whether extraction succeeded */ + success: boolean; + /** Number of memories extracted */ + extractedCount: number; + /** Number of memories actually stored (after dedup) */ + storedCount: number; + /** The memory records that were stored */ + records: MemoryRecord[]; + /** Scene names detected during extraction */ + sceneNames: string[]; + /** Last scene name (for continuity in next extraction) */ + lastSceneName?: string; +} + +// ============================ +// Core function +// ============================ + +/** + * Run the full L1 extraction pipeline on conversation messages. + * + * @param messages - Filtered conversation messages (from L0 or directly from hook) + * @param sessionKey - The session key + * @param baseDir - Base data directory (~/.openclaw/memory-tdai/) + * @param config - OpenClaw config (for LLM access) + * @param options - Extraction options + * @param logger - Optional logger + */ +export async function extractL1Memories(params: { + messages: ConversationMessage[]; + sessionKey: string; + sessionId?: string; + baseDir: string; + config: unknown; + options?: { + /** Max new messages to send in one extraction call */ + maxMessagesPerExtraction?: number; + /** Max background messages for context */ + maxBackgroundMessages?: number; + /** Enable conflict detection */ + enableDedup?: boolean; + /** Max memories extracted per call */ + maxMemoriesPerSession?: number; + /** LLM model override */ + model?: string; + /** Previous scene name for continuity */ + previousSceneName?: string; + /** Vector store for cosine similarity candidate recall */ + vectorStore?: VectorStore; + /** Embedding service for computing query vectors */ + embeddingService?: EmbeddingService; + /** Top-K candidates for conflict recall (default: 5) */ + conflictRecallTopK?: number; + }; + logger?: Logger; + /** Plugin instance ID for metric reporting (optional — metrics skipped if absent) */ + instanceId?: string; +}): Promise { + const { messages, sessionKey, sessionId, baseDir, config, logger, instanceId: metricInstanceId } = params; + const options = params.options ?? {}; + const maxNewMessages = options.maxMessagesPerExtraction ?? 10; + const maxBgMessages = options.maxBackgroundMessages ?? 5; + const enableDedup = options.enableDedup ?? true; + const maxMemoriesPerSession = options.maxMemoriesPerSession ?? 10; + + if (messages.length === 0) { + logger?.debug?.(`${TAG} No messages to extract from`); + return { success: true, extractedCount: 0, storedCount: 0, records: [], sceneNames: [] }; + } + + const l1StartMs = Date.now(); + + // Quality gate: filter messages through L1 extraction rules (length, symbols, + // prompt injection, etc.) before sending to the LLM. L0 deliberately captures + // everything; the strict filtering happens here at L1 stage. + const qualifiedMessages = messages.filter((m) => shouldExtractL1(m.content)); + if (qualifiedMessages.length < messages.length) { + logger?.debug?.( + `${TAG} L1 quality filter: ${messages.length} → ${qualifiedMessages.length} messages ` + + `(${messages.length - qualifiedMessages.length} filtered out)`, + ); + } + + if (qualifiedMessages.length === 0) { + logger?.debug?.(`${TAG} All messages filtered out by L1 quality gate`); + return { success: true, extractedCount: 0, storedCount: 0, records: [], sceneNames: [] }; + } + + // Split messages into background (older) + new (recent) + const newMessages = qualifiedMessages.slice(-maxNewMessages); + const bgEndIdx = qualifiedMessages.length - newMessages.length; + const backgroundMessages = bgEndIdx > 0 + ? qualifiedMessages.slice(Math.max(0, bgEndIdx - maxBgMessages), bgEndIdx) + : []; + + logger?.debug?.(`${TAG} Extracting from ${newMessages.length} new messages (+ ${backgroundMessages.length} background) [${qualifiedMessages.length} qualified from ${messages.length} input]`); + + // Step 1: LLM extraction (scene segmentation + memory extraction) + let scenes: SceneSegment[]; + try { + scenes = await callLlmExtraction({ + newMessages, + backgroundMessages, + previousSceneName: options.previousSceneName, + config, + logger, + model: options.model, + }); + logger?.debug?.(`${TAG} LLM detected ${scenes.length} scene(s)`); + } catch (err) { + logger?.error(`${TAG} LLM extraction failed: ${err instanceof Error ? err.message : String(err)}`); + return { success: false, extractedCount: 0, storedCount: 0, records: [], sceneNames: [] }; + } + + // Flatten all memories across scenes + const allExtracted: ExtractedMemory[] = []; + const sceneNames: string[] = []; + + for (const scene of scenes) { + sceneNames.push(scene.scene_name); + for (const mem of scene.memories) { + const memType = normalizeType(mem.type); + if (!memType) { + logger?.warn?.(`${TAG} Skipping memory with invalid type "${mem.type}"`); + continue; + } + allExtracted.push({ + content: mem.content, + type: memType, + priority: typeof mem.priority === "number" ? mem.priority : 50, + source_message_ids: Array.isArray(mem.source_message_ids) ? mem.source_message_ids : [], + metadata: mem.metadata ?? {}, + scene_name: scene.scene_name, + }); + } + } + + logger?.debug?.(`${TAG} Total extracted memories: ${allExtracted.length} across ${scenes.length} scene(s)`); + + if (allExtracted.length === 0) { + return { + success: true, + extractedCount: 0, + storedCount: 0, + records: [], + sceneNames, + lastSceneName: sceneNames[sceneNames.length - 1], + }; + } + + // Limit per session + let extracted = allExtracted; + if (extracted.length > maxMemoriesPerSession) { + logger?.debug?.(`${TAG} Limiting from ${extracted.length} to ${maxMemoriesPerSession} memories per session`); + extracted = extracted.slice(0, maxMemoriesPerSession); + } + + // Assign temporary IDs to extracted memories (needed for batch dedup) + const memoriesWithIds = extracted.map((m) => ({ + ...m, + record_id: generateMemoryId(), + })); + + // Step 2: Batch Conflict Detection + Write + let storedRecords: MemoryRecord[]; + + if (enableDedup) { + try { + const decisions = await batchDedup({ + memories: memoriesWithIds, + config, + logger, + model: options.model, + vectorStore: options.vectorStore, + embeddingService: options.embeddingService, + conflictRecallTopK: options.conflictRecallTopK, + }); + + storedRecords = await applyDecisions({ + memoriesWithIds, + decisions, + baseDir, + sessionKey, + sessionId, + logger, + vectorStore: options.vectorStore, + embeddingService: options.embeddingService, + }); + } catch (err) { + logger?.warn?.(`${TAG} Batch dedup failed, storing all as new: ${err instanceof Error ? err.message : String(err)}`); + storedRecords = await storeAllDirectly(memoriesWithIds, baseDir, sessionKey, sessionId, logger, options.vectorStore, options.embeddingService); + } + } else { + storedRecords = await storeAllDirectly(memoriesWithIds, baseDir, sessionKey, sessionId, logger, options.vectorStore, options.embeddingService); + } + + logger?.info(`${TAG} Extraction complete: extracted=${extracted.length}, stored=${storedRecords.length}`); + + // ── l1_extraction metric ── + if (metricInstanceId && logger) { + // Build type distribution of stored memories + const memoriesByType: Record = {}; + for (const r of storedRecords) { + memoriesByType[r.type] = (memoriesByType[r.type] ?? 0) + 1; + } + report("l1_extraction", { + sessionKey, + inputMessageCount: messages.length, + memoriesExtracted: extracted.length, + memoriesStored: storedRecords.length, + memoriesStoredContent: storedRecords.map((r) => ({ + content: r.content, + type: r.type, + scene: r.scene_name ?? null, + })), + memoriesByType, + totalDurationMs: Date.now() - l1StartMs, + success: true, + error: null, + }); + } + + return { + success: true, + extractedCount: extracted.length, + storedCount: storedRecords.length, + records: storedRecords, + sceneNames, + lastSceneName: sceneNames[sceneNames.length - 1], + }; +} + +// ============================ +// LLM call +// ============================ + +/** + * Call LLM to extract scene-segmented memories from conversation messages. + */ +async function callLlmExtraction(params: { + newMessages: ConversationMessage[]; + backgroundMessages: ConversationMessage[]; + previousSceneName?: string; + config: unknown; + logger?: Logger; + model?: string; +}): Promise { + const { newMessages, backgroundMessages, previousSceneName, config, logger, model } = params; + + const runner = new CleanContextRunner({ + config, + modelRef: model, + enableTools: false, + logger, + }); + + const userPrompt = formatExtractionPrompt({ + newMessages, + backgroundMessages, + previousSceneName, + }); + + const result = await runner.run({ + prompt: userPrompt, + systemPrompt: EXTRACT_MEMORIES_SYSTEM_PROMPT, + taskId: "l1-extraction", + timeoutMs: 180_000, + // maxTokens: 4000, + }); + + return parseExtractionResult(result, logger); +} + +/** + * Parse the LLM's JSON response into SceneSegment array. + * Expected format: [{scene_name, message_ids, memories: [...]}] + */ +function parseExtractionResult(raw: string, logger?: Logger): SceneSegment[] { + try { + // Strip markdown code block wrappers if present + let cleaned = raw.trim(); + if (cleaned.startsWith("```")) { + cleaned = cleaned.replace(/^```(?:json)?\s*\n?/, "").replace(/\n?```\s*$/, ""); + } + + // Try to extract JSON array + const arrayMatch = cleaned.match(/\[[\s\S]*\]/); + if (!arrayMatch) { + logger?.warn?.(`${TAG} No JSON array found in extraction response`); + return []; + } + + // Sanitize control characters inside JSON string literals that LLM may produce + const sanitized = sanitizeJsonForParse(arrayMatch[0]); + const parsed = JSON.parse(sanitized) as unknown[]; + + if (!Array.isArray(parsed)) { + logger?.warn?.(`${TAG} Extraction response is not an array`); + return []; + } + + const scenes: SceneSegment[] = []; + for (const item of parsed) { + if (!item || typeof item !== "object") continue; + const s = item as Record; + + scenes.push({ + scene_name: typeof s.scene_name === "string" ? s.scene_name : "未知情境", + message_ids: Array.isArray(s.message_ids) ? s.message_ids.map(String) : [], + memories: Array.isArray(s.memories) + ? (s.memories as Array>) + .filter((m) => m && typeof m === "object" && typeof m.content === "string" && (m.content as string).length > 0) + .map((m) => ({ + content: String(m.content), + type: String(m.type ?? "episodic"), + priority: typeof m.priority === "number" ? m.priority : 50, + source_message_ids: Array.isArray(m.source_message_ids) ? m.source_message_ids.map(String) : [], + metadata: (m.metadata && typeof m.metadata === "object" ? m.metadata : {}) as Record, + })) + : [], + }); + } + + return scenes; + } catch (err) { + logger?.warn?.(`${TAG} Failed to parse extraction result: ${err instanceof Error ? err.message : String(err)}`); + return []; + } +} + +// ============================ +// Write helpers +// ============================ + +/** + * Apply batch dedup decisions — write memories according to their decisions. + */ +async function applyDecisions(params: { + memoriesWithIds: Array; + decisions: DedupDecision[]; + baseDir: string; + sessionKey: string; + sessionId?: string; + logger?: Logger; + vectorStore?: VectorStore; + embeddingService?: EmbeddingService; +}): Promise { + const { memoriesWithIds, decisions, baseDir, sessionKey, sessionId, logger, vectorStore, embeddingService } = params; + const storedRecords: MemoryRecord[] = []; + + // Build a map from record_id → decision + const decisionMap = new Map(); + for (const d of decisions) { + decisionMap.set(d.record_id, d); + } + + for (const memoryWithId of memoriesWithIds) { + const decision = decisionMap.get(memoryWithId.record_id) ?? { + record_id: memoryWithId.record_id, + action: "store" as const, + target_ids: [], + }; + + try { + const record = await writeMemory({ + memory: memoryWithId, + decision, + baseDir, + sessionKey, + sessionId, + logger, + vectorStore, + embeddingService, + }); + + if (record) { + storedRecords.push(record); + } + } catch (err) { + logger?.warn?.( + `${TAG} Write failed for memory "${memoryWithId.content.slice(0, 50)}...": ${err instanceof Error ? err.message : String(err)}`, + ); + } + } + + return storedRecords; +} + +/** + * Store all memories directly (no dedup). + */ +async function storeAllDirectly( + memoriesWithIds: Array, + baseDir: string, + sessionKey: string, + sessionId: string | undefined, + logger?: Logger, + vectorStore?: VectorStore, + embeddingService?: EmbeddingService, +): Promise { + const storedRecords: MemoryRecord[] = []; + + for (const memoryWithId of memoriesWithIds) { + try { + const record = await writeMemory({ + memory: memoryWithId, + decision: { + record_id: memoryWithId.record_id, + action: "store", + target_ids: [], + }, + baseDir, + sessionKey, + sessionId, + logger, + vectorStore, + embeddingService, + }); + if (record) { + storedRecords.push(record); + } + } catch (err) { + logger?.warn?.( + `${TAG} Write failed for memory "${memoryWithId.content.slice(0, 50)}...": ${err instanceof Error ? err.message : String(err)}`, + ); + } + } + + return storedRecords; +} + +// ============================ +// Helpers +// ============================ + +const VALID_TYPES: MemoryType[] = ["persona", "episodic", "instruction"]; + +function normalizeType(raw: string): MemoryType | null { + const lower = raw.toLowerCase().trim(); + if (VALID_TYPES.includes(lower as MemoryType)) { + return lower as MemoryType; + } + // Handle legacy type names + if (lower === "episode") return "episodic"; + if (lower === "instruct") return "instruction"; + if (lower === "preference") return "persona"; // fold preference into persona + return null; +} diff --git a/src/record/l1-reader.ts b/src/record/l1-reader.ts new file mode 100644 index 0000000..0b34554 --- /dev/null +++ b/src/record/l1-reader.ts @@ -0,0 +1,218 @@ +/** + * L1 Memory Reader: reads persisted L1 memory records. + * + * Provides two data paths: + * + * 1. **SQLite** (preferred): `queryMemoryRecords()` — uses VectorStore's `queryL1Records()` + * with composite indexes on (session_key, updated_time) and (session_id, updated_time) + * for efficient session-scoped and time-range queries. + * + * 2. **JSONL** (fallback): `readMemoryRecords()` / `readAllMemoryRecords()` — reads from + * `records/YYYY-MM-DD.jsonl` files. Used when VectorStore is unavailable or degraded. + */ + +import fs from "node:fs/promises"; +import path from "node:path"; +import type { MemoryRecord, MemoryType, EpisodicMetadata } from "./l1-writer.js"; +import type { VectorStore, L1RecordRow, L1QueryFilter } from "../store/vector-store.js"; + +// Re-export types that readers need +export type { MemoryRecord, MemoryType, EpisodicMetadata } from "./l1-writer.js"; +export type { L1QueryFilter } from "../store/vector-store.js"; + +interface Logger { + debug?: (message: string) => void; + info: (message: string) => void; + warn: (message: string) => void; + error: (message: string) => void; +} + +const TAG = "[memory-tdai] [l1-reader]"; + +// ============================ +// SQLite-based queries (preferred) +// ============================ + +/** + * Query L1 memory records from SQLite via VectorStore. + * + * This is the **preferred** read path — it uses the composite index + * `idx_l1_session_updated(session_id, updated_time)` for efficient + * session-scoped and time-range queries. + * + * All timestamps are UTC ISO 8601 (as stored by l1-writer's dual-write). + * + * Falls back to empty array if VectorStore is null or degraded. + */ +export function queryMemoryRecords( + vectorStore: VectorStore | null | undefined, + filter?: L1QueryFilter, + logger?: Logger, +): MemoryRecord[] { + if (!vectorStore) { + logger?.warn(`${TAG} queryMemoryRecords: no VectorStore available, returning empty`); + return []; + } + + const rows = vectorStore.queryL1Records(filter); + return rows.map(rowToMemoryRecord); +} + +/** + * Convert a raw SQLite L1RecordRow to a MemoryRecord (same shape as JSONL records). + */ +function rowToMemoryRecord(row: L1RecordRow): MemoryRecord { + let metadata: EpisodicMetadata | Record = {}; + try { + metadata = JSON.parse(row.metadata_json) as EpisodicMetadata | Record; + } catch { + // malformed JSON — use empty object + } + + // Reconstruct timestamps array from timestamp_start / timestamp_end + const timestamps: string[] = []; + if (row.timestamp_str) timestamps.push(row.timestamp_str); + if (row.timestamp_start && row.timestamp_start !== row.timestamp_str) timestamps.push(row.timestamp_start); + if (row.timestamp_end && row.timestamp_end !== row.timestamp_str && row.timestamp_end !== row.timestamp_start) { + timestamps.push(row.timestamp_end); + } + + return { + id: row.record_id, + content: row.content, + type: row.type as MemoryType, + priority: row.priority, + scene_name: row.scene_name, + source_message_ids: [], // not stored in SQLite (vector search doesn't need them) + metadata, + timestamps, + createdAt: row.created_time, + updatedAt: row.updated_time, + sessionKey: row.session_key, + sessionId: row.session_id, + }; +} + +// ============================ +// JSONL-based reads (fallback) +// ============================ + +/** + * Read all memory records for a session from JSONL files. + * + * Current naming mode: + * - Daily merged file: records/YYYY-MM-DD.jsonl (all sessions in one file) + */ +export async function readMemoryRecords( + sessionKey: string, + baseDir: string, + logger?: Logger, +): Promise { + const recordsDir = path.join(baseDir, "records"); + const dateFilePattern = /^\d{4}-\d{2}-\d{2}\.jsonl$/; + + let entries: import("node:fs").Dirent[]; + try { + entries = await fs.readdir(recordsDir, { withFileTypes: true }); + } catch { + // Directory doesn't exist yet + return []; + } + + const targetFiles = entries + .filter((entry) => entry.isFile() && dateFilePattern.test(entry.name)) + .map((entry) => entry.name) + .sort(); + + if (targetFiles.length === 0) { + return []; + } + + const records: MemoryRecord[] = []; + + for (const fileName of targetFiles) { + const filePath = path.join(recordsDir, fileName); + + let raw: string; + try { + raw = await fs.readFile(filePath, "utf-8"); + } catch { + logger?.warn?.(`${TAG} Failed to read L1 file: ${filePath}`); + continue; + } + + const lines = raw.split("\n").filter((line) => line.trim()); + for (let i = 0; i < lines.length; i++) { + const line = lines[i]; + try { + const parsed = JSON.parse(line) as Partial; + if (parsed.sessionKey !== sessionKey) { + continue; + } + records.push(parsed as MemoryRecord); + } catch { + logger?.warn?.(`${TAG} Skipping malformed JSONL line in ${filePath}:${i + 1}`); + } + } + } + + records.sort((a, b) => { + const ta = a.updatedAt || a.createdAt || ""; + const tb = b.updatedAt || b.createdAt || ""; + return ta.localeCompare(tb); + }); + + return records; +} + +/** + * Read ALL memory records across all session JSONL files. + */ +export async function readAllMemoryRecords( + baseDir: string, + logger?: Logger, +): Promise { + const recordsDir = path.join(baseDir, "records"); + try { + const files = await fs.readdir(recordsDir); + const allRecords: MemoryRecord[] = []; + + for (const file of files) { + if (!file.endsWith(".jsonl")) continue; + const filePath = path.join(recordsDir, file); + try { + const raw = await fs.readFile(filePath, "utf-8"); + const lines = raw.split("\n").filter((line: string) => line.trim()); + for (const line of lines) { + try { + allRecords.push(JSON.parse(line) as MemoryRecord); + } catch { + logger?.warn?.(`${TAG} Skipping malformed JSONL line in ${file}`); + } + } + } catch { + logger?.warn?.(`${TAG} Failed to read ${file}`); + } + } + + allRecords.sort((a, b) => { + const ta = a.updatedAt || a.createdAt || ""; + const tb = b.updatedAt || b.createdAt || ""; + return ta.localeCompare(tb); + }); + + return allRecords; + + } catch { + // records/ directory doesn't exist yet + return []; + } +} + +// ============================ +// Helpers +// ============================ + +function sanitizeFilename(name: string): string { + return name.replace(/[<>:"/\\|?*\x00-\x1f]/g, "_"); +} diff --git a/src/record/l1-writer.ts b/src/record/l1-writer.ts new file mode 100644 index 0000000..d9510ef --- /dev/null +++ b/src/record/l1-writer.ts @@ -0,0 +1,280 @@ +/** + * L1 Memory Writer: writes extracted memories to JSONL files. + * + * File naming: records/YYYY-MM-DD.jsonl (daily shards, all sessions merged). + * Each record includes sessionKey for traceability. + * + * Write strategy: + * - JSONL is the append-only persistent store (source of truth for backup/recovery). + * - VectorStore (SQLite) is the primary retrieval engine. + * - On update/merge, old records are deleted from VectorStore in real-time; + * JSONL is append-only and cleaned up periodically by memory-cleaner. + * + * Supports store (append), update, merge, and skip operations. + * + * v3: Aligned with Kenty's prompt output format — 3 memory types (persona/episodic/instruction), + * numeric priority, scene_name, source_message_ids, metadata, timestamps. + */ + +import fs from "node:fs/promises"; +import path from "node:path"; +import crypto from "node:crypto"; +import type { VectorStore } from "../store/vector-store.js"; +import type { EmbeddingService } from "../store/embedding.js"; + +// ============================ +// Types +// ============================ + +/** v3: 3 memory types aligned with Kenty's extraction prompt */ +export type MemoryType = "persona" | "episodic" | "instruction"; + +/** Metadata for episodic memories (activity time range) */ +export interface EpisodicMetadata { + activity_start_time?: string; // ISO 8601 + activity_end_time?: string; // ISO 8601 +} + +/** + * A persisted memory record in L1 JSONL files. + * + * v3 changes from v2: + * - `importance: "high"|"medium"|"low"` → `priority: number` (0-100, -1 for strict global instructions) + * - Added `scene_name`, `source_message_ids`, `metadata`, `timestamps` + * - Removed `keywords` (will be rebuilt from content for search) + * - MemoryType reduced from 4 to 3 (removed "preference", folded into "persona") + */ +export interface MemoryRecord { + /** Unique ID for dedup updates */ + id: string; + /** Memory content */ + content: string; + /** Memory type: persona / episodic / instruction */ + type: MemoryType; + /** Priority score: 0-100 (higher = more important), -1 = strict global instruction */ + priority: number; + /** Scene name this memory belongs to */ + scene_name: string; + /** Source message IDs that contributed to this memory */ + source_message_ids: string[]; + /** Type-specific metadata (e.g., activity_start_time for episodic) */ + metadata: EpisodicMetadata | Record; + /** Timestamp trail: all timestamps related to this memory (for merge history tracking) */ + timestamps: string[]; + /** Creation timestamp (ISO) */ + createdAt: string; + /** Last update timestamp (ISO) */ + updatedAt: string; + /** Source session key (conversation channel identifier) */ + sessionKey: string; + /** Source session ID (single conversation instance identifier) */ + sessionId: string; +} + +/** + * A memory as extracted by LLM (before dedup / persistence). + * Matches the output format of Kenty's extraction prompt. + */ +export interface ExtractedMemory { + content: string; + type: MemoryType; + priority: number; + source_message_ids: string[]; + metadata: EpisodicMetadata | Record; + /** Scene name this memory was extracted in */ + scene_name: string; +} + +export type DedupAction = "store" | "update" | "merge" | "skip"; + +/** + * v3 batch dedup decision — one per new memory, aligned with Kenty's conflict detection prompt. + * + * Key changes: + * - `targetId` → `target_ids` (array, supports multi-target merge/update) + * - Added `merged_type`, `merged_priority`, `merged_timestamps` for cross-type merge + */ +export interface DedupDecision { + /** Which new memory this decision is about */ + record_id: string; + action: DedupAction; + /** IDs of existing records to replace/remove (for update/merge) */ + target_ids: string[]; + /** Merged/updated content text (for update/merge) */ + merged_content?: string; + /** Best type after merge (for update/merge, may differ from original) */ + merged_type?: MemoryType; + /** Priority after merge (for update/merge) */ + merged_priority?: number; + /** Union of all related timestamps (for update/merge) */ + merged_timestamps?: string[]; +} + +interface Logger { + debug?: (message: string) => void; + info: (message: string) => void; + warn: (message: string) => void; + error: (message: string) => void; +} + +const TAG = "[memory-tdai][l1-writer]"; + +// ============================ +// Core functions +// ============================ + +/** + * Generate a unique memory ID. + */ +export function generateMemoryId(): string { + return `m_${Date.now()}_${crypto.randomBytes(4).toString("hex")}`; +} + +/** + * Write a memory record according to the dedup decision. + * + * - store: append new record + * - update: remove target records + append updated record + * - merge: remove target records + append merged record + * - skip: do nothing + * + * v3: supports multi-target removal for update/merge. + * v3.1: optional VectorStore + EmbeddingService for dual-write (JSONL + vector). + */ +export async function writeMemory(params: { + memory: ExtractedMemory; + decision: DedupDecision; + baseDir: string; + sessionKey: string; + sessionId?: string; + logger?: Logger; + /** Optional vector store for dual-write (JSONL + vector DB) */ + vectorStore?: VectorStore; + /** Optional embedding service (required when vectorStore is provided) */ + embeddingService?: EmbeddingService; +}): Promise { + const { memory, decision, baseDir, sessionKey, sessionId, logger, vectorStore, embeddingService } = params; + + if (decision.action === "skip") { + logger?.debug?.(`${TAG} Skipping memory: ${memory.content.slice(0, 50)}...`); + return null; + } + + const now = new Date().toISOString(); + + // Determine final content, type, priority based on action + let finalContent: string; + let finalType: MemoryType; + let finalPriority: number; + let finalTimestamps: string[]; + + if (decision.action === "merge" || decision.action === "update") { + finalContent = decision.merged_content ?? memory.content; + finalType = decision.merged_type ?? memory.type; + finalPriority = decision.merged_priority ?? memory.priority; + finalTimestamps = decision.merged_timestamps ?? [now]; + } else { + // store + finalContent = memory.content; + finalType = memory.type; + finalPriority = memory.priority; + finalTimestamps = [now]; + } + + const record: MemoryRecord = { + id: decision.record_id || generateMemoryId(), + content: finalContent, + type: finalType, + priority: finalPriority, + scene_name: memory.scene_name, + source_message_ids: memory.source_message_ids, + metadata: memory.metadata, + timestamps: finalTimestamps, + createdAt: now, + updatedAt: now, + sessionKey, + sessionId: sessionId || "", + }; + + const recordsDir = path.join(baseDir, "records"); + await fs.mkdir(recordsDir, { recursive: true }); + + const shardDate = formatLocalDate(new Date()); + const filePath = path.join(recordsDir, `${shardDate}.jsonl`); + + if ((decision.action === "update" || decision.action === "merge") && decision.target_ids.length > 0) { + // Remove target records from VectorStore (real-time deletion for retrieval accuracy). + // JSONL is append-only — old records remain in files and are cleaned up periodically + // by memory-cleaner (which reconciles against VectorStore as source of truth). + if (vectorStore) { + try { + vectorStore.deleteBatch(decision.target_ids); + logger?.debug?.(`${TAG} VectorStore: deleted ${decision.target_ids.length} target record(s) for ${decision.action}`); + } catch (err) { + logger?.warn?.( + `${TAG} VectorStore delete failed for ${decision.action}: ${err instanceof Error ? err.message : String(err)}`, + ); + } + } + await fs.appendFile(filePath, JSON.stringify(record) + "\n", "utf-8"); + logger?.debug?.(`${TAG} ${decision.action} memory: removed [${decision.target_ids.join(",")}] from VectorStore → ${record.id}: ${finalContent.slice(0, 80)}...`); + } else { + // store: append a new line + await fs.appendFile(filePath, JSON.stringify(record) + "\n", "utf-8"); + logger?.debug?.(`${TAG} Stored memory ${record.id}: ${finalContent.slice(0, 80)}...`); + } + + // === Vector Store dual-write === + if (vectorStore) { + try { + logger?.debug?.( + `${TAG} [vec-dual-write] START id=${record.id}, contentLen=${record.content.length}, ` + + `content="${record.content.slice(0, 80)}..."`, + ); + + let embedding: Float32Array | undefined; + + if (embeddingService) { + try { + embedding = await embeddingService.embed(record.content); + logger?.debug?.( + `${TAG} [vec-dual-write] Embedding OK: dims=${embedding.length}, ` + + `norm=${Math.sqrt(Array.from(embedding).reduce((s, v) => s + v * v, 0)).toFixed(4)}`, + ); + } catch (embedErr) { + // Embedding failed — pass undefined to upsert() which writes + // metadata + FTS only, skipping the vec0 table. + logger?.warn( + `${TAG} [vec-dual-write] Embedding FAILED for id=${record.id}, ` + + `will write metadata only: ${embedErr instanceof Error ? embedErr.message : String(embedErr)}`, + ); + } + } + + const upsertOk = vectorStore.upsert(record, embedding); + logger?.debug?.(`${TAG} [vec-dual-write] upsert result=${upsertOk} id=${record.id}`); + } catch (err) { + // Vector write failure should NOT block the main JSONL write + logger?.warn?.( + `${TAG} [vec-dual-write] FAILED (JSONL already written) id=${record.id}: ${err instanceof Error ? err.message : String(err)}`, + ); + } + } else { + logger?.debug?.( + `${TAG} [vec-dual-write] SKIPPED id=${record.id}: vectorStore=${!!vectorStore}`, + ); + } + + return record; +} + +// ============================ +// Helpers +// ============================ + +function formatLocalDate(d: Date): string { + const y = d.getFullYear(); + const m = String(d.getMonth() + 1).padStart(2, "0"); + const day = String(d.getDate()).padStart(2, "0"); + return `${y}-${m}-${day}`; +} \ No newline at end of file diff --git a/src/report/reporter.ts b/src/report/reporter.ts new file mode 100644 index 0000000..b0091a4 --- /dev/null +++ b/src/report/reporter.ts @@ -0,0 +1,95 @@ +import { randomUUID } from "node:crypto"; +import fs from "node:fs/promises"; +import path from "node:path"; + +export const REPORT_CONST = { + PLUGIN: "plugin", +} as const; + +export type ReportPayload = Record; + +export interface IReporter { + reportFunc(category: string, payload: ReportPayload): void; +} + +// ── Singleton ── + +let _reporter: IReporter | undefined; + +export function initReporter(opts: { + enabled: boolean; + type: string; + logger: { info: (msg: string) => void }; + instanceId: string; + pluginVersion: string; +}): void { + if (_reporter) return; + if (!opts.enabled) return; + switch (opts.type) { + case "local": + _reporter = new LocalReporter(opts.logger, opts.instanceId, opts.pluginVersion); + break; + // TODO: add new reporter type + default: + opts.logger.info(`[memory-tdai] Unknown reporter type "${opts.type}", disabled reporting`); + break; + } +} + +export function setReporter(reporter: IReporter): void { + _reporter = reporter; +} + +export function report(event: string, data: ReportPayload): void { + if (!_reporter) return; + try { + _reporter.reportFunc(REPORT_CONST.PLUGIN, { event, ...data }); + } catch { /* never block business logic */ } +} + +// ── LocalReporter (default) ── + +class LocalReporter implements IReporter { + constructor( + private readonly logger: { info: (msg: string) => void }, + private readonly instanceId: string, + private readonly pluginVersion: string, + ) {} + + reportFunc(category: string, payload: ReportPayload): void { + try { + this.logger.info(JSON.stringify({ + tag: "METRIC", + category, + plugin: "memory-tdai", + instanceId: this.instanceId, + pluginVersion: this.pluginVersion, + ts: new Date().toISOString(), + ...payload, + })); + } catch { /* swallow */ } + } +} + +// ── Instance ID (persisted per-install) ── + +let _instanceIdCache: string | undefined; + +export async function getOrCreateInstanceId(pluginDataDir: string): Promise { + if (_instanceIdCache) return _instanceIdCache; + + const idFile = path.join(pluginDataDir, ".metadata", "instance_id"); + try { + const existing = (await fs.readFile(idFile, "utf-8")).trim(); + if (existing) { + _instanceIdCache = existing; + return existing; + } + } catch { /* file doesn't exist */ } + + const newId = randomUUID(); + await fs.mkdir(path.dirname(idFile), { recursive: true }); + await fs.writeFile(idFile, newId, "utf-8"); + _instanceIdCache = newId; + return newId; +} diff --git a/src/scene/scene-extractor.ts b/src/scene/scene-extractor.ts new file mode 100644 index 0000000..f900472 --- /dev/null +++ b/src/scene/scene-extractor.ts @@ -0,0 +1,416 @@ +/** + * SceneExtractor: LLM-driven memory extraction into scene blocks. + * + * Replaces the keyword-based SceneManager.processNewMemories() with an + * LLM agent that autonomously reads/writes scene block files using tools. + * + * Security: The LLM is sandboxed — workspaceDir is set to scene_blocks/ + * so it can ONLY operate on .md scene files. System files (checkpoint, + * scene_index, persona.md) are physically invisible to the LLM. + * + * Flow: + * 1. Backup + load scene index + build summaries + * 2. Assemble extraction prompt with memories + scene context + * 3. Run via CleanContextRunner (tools enabled, sandboxed to scene_blocks/) + * 4. Cleanup: remove soft-deletes, sync index, update navigation + * 5. Parse LLM text output for out-of-band persona update signals + */ + +import fs from "node:fs/promises"; +import path from "node:path"; +import { CleanContextRunner } from "../utils/clean-context-runner.js"; +import { CheckpointManager } from "../utils/checkpoint.js"; +import { BackupManager } from "../utils/backup.js"; +import { readSceneIndex, syncSceneIndex } from "../scene/scene-index.js"; +import type { SceneIndexEntry } from "../scene/scene-index.js"; +import { parseSceneBlock } from "../scene/scene-format.js"; +import { generateSceneNavigation, stripSceneNavigation } from "../scene/scene-navigation.js"; +import { buildSceneExtractionPrompt } from "../prompts/scene-extraction.js"; +import { report } from "../report/reporter.js"; + +const TAG = "[memory-tdai] [extractor]"; + +interface ExtractorLogger { + debug?: (message: string) => void; + info: (message: string) => void; + warn: (message: string) => void; + error: (message: string) => void; +} + +export interface ExtractionResult { + memoriesProcessed: number; + success: boolean; + error?: string; +} + +export interface SceneExtractorOptions { + dataDir: string; + config: unknown; + model?: string; + maxScenes?: number; + sceneBackupCount?: number; + timeoutMs?: number; + logger?: ExtractorLogger; + /** Plugin instance ID for metric reporting (optional) */ + instanceId?: string; +} + +/** + * Parse LLM text output for a persona update request signal. + * + * Supports multiple formats for robustness: + * - Block: [PERSONA_UPDATE_REQUEST]reason: xxx[/PERSONA_UPDATE_REQUEST] + * - Inline: PERSONA_UPDATE_REQUEST: xxx + */ +export function parsePersonaUpdateSignal(text: string): { reason: string } | null { + // Block format: [PERSONA_UPDATE_REQUEST]...[/PERSONA_UPDATE_REQUEST] + const blockMatch = text.match( + /\[PERSONA_UPDATE_REQUEST\]\s*(?:reason:\s*)?(.+?)\s*\[\/PERSONA_UPDATE_REQUEST\]/s, + ); + if (blockMatch) return { reason: blockMatch[1]!.trim() }; + + // Inline format: PERSONA_UPDATE_REQUEST: reason text + const inlineMatch = text.match( + /PERSONA_UPDATE_REQUEST:\s*(.+?)(?:\n|$)/, + ); + if (inlineMatch) return { reason: inlineMatch[1]!.trim() }; + + return null; +} + +export class SceneExtractor { + private dataDir: string; + private runner: CleanContextRunner; + private maxScenes: number; + private sceneBackupCount: number; + private timeoutMs: number; + private logger: ExtractorLogger | undefined; + private instanceId: string | undefined; + + constructor(opts: SceneExtractorOptions) { + this.dataDir = opts.dataDir; + this.maxScenes = opts.maxScenes ?? 20; + this.sceneBackupCount = opts.sceneBackupCount ?? 10; + this.timeoutMs = opts.timeoutMs ?? 300_000; // 5 min — LLM may do multiple tool calls + this.logger = opts.logger; + this.instanceId = opts.instanceId; + + this.runner = new CleanContextRunner({ + config: opts.config, + modelRef: opts.model, + enableTools: true, + logger: opts.logger, + }); + + this.logger?.debug?.(`${TAG} Created: dataDir=${opts.dataDir}, model=${opts.model ?? "(default)"}, maxScenes=${this.maxScenes}, timeout=${this.timeoutMs}ms`); + } + + /** + * Extract a batch of memories into scene blocks using the LLM agent. + * + * @param memories - Array of raw memory records from the API + * @returns Extraction result with count and success flag + */ + async extract(memories: Array<{ content: string; created_at: string; id?: string }>): Promise { + const extractStartMs = Date.now(); + this.logger?.info(`${TAG} extract() start: ${memories.length} memories`); + + if (memories.length === 0) { + this.logger?.debug?.(`${TAG} extract() skipped: no memories`); + return { memoriesProcessed: 0, success: true }; + } + + const sceneBlocksDir = path.join(this.dataDir, "scene_blocks"); + const metadataDir = path.join(this.dataDir, ".metadata"); + + // Ensure directories exist + await fs.mkdir(sceneBlocksDir, { recursive: true }); + await fs.mkdir(metadataDir, { recursive: true }); + + // Phase 1: Backup + const backupStartMs = Date.now(); + const cpManager = new CheckpointManager(this.dataDir); + const cp = await cpManager.read(); + const bm = new BackupManager(path.join(this.dataDir, ".backup")); + await bm.backupDirectory(sceneBlocksDir, "scene_blocks", `offset${cp.total_processed}`, this.sceneBackupCount); + this.logger?.debug?.(`${TAG} extract() backup phase: ${Date.now() - backupStartMs}ms`); + + // Phase 2: Load scene index + const indexStartMs = Date.now(); + const index = await readSceneIndex(this.dataDir); + this.logger?.debug?.(`${TAG} extract() scene index loaded: ${index.length} entries (${Date.now() - indexStartMs}ms)`); + + // Build scene summaries for the prompt (relative filenames only) + const { summaries: sceneSummaries, filenames: existingSceneFiles } = + this.buildSceneSummaries(index); + + // Build scene count warning (tiered system) + let sceneCountWarning: string | undefined; + const sceneCount = index.length; + if (sceneCount >= this.maxScenes) { + sceneCountWarning = `当前场景数量为 **${sceneCount} 个**，已达到或超过 ${this.maxScenes} 个上限！\n**你必须先执行 MERGE 操作**，将最相似的 2-4 个场景合并为 1 个，然后再处理新记忆。\n参考合并对象：热度最低或主题高度重叠的场景。`; + this.logger?.warn(`${TAG} extract() scene count at limit: ${sceneCount}/${this.maxScenes}`); + } else if (sceneCount === this.maxScenes - 1) { + sceneCountWarning = `当前场景数量为 **${sceneCount} 个**，距离上限只差 1 个！\n本次处理**只能 UPDATE 现有场景，不能 CREATE 新场景**。`; + this.logger?.warn(`${TAG} extract() scene count near limit (CREATE blocked): ${sceneCount}/${this.maxScenes}`); + } else if (sceneCount >= this.maxScenes - 3) { + sceneCountWarning = `当前场景数量为 **${sceneCount} 个**，建议优先考虑 UPDATE 或主动 MERGE 相似场景。`; + this.logger?.debug?.(`${TAG} extract() scene count approaching limit: ${sceneCount}/${this.maxScenes}`); + } + + // Snapshot scene index + content before LLM — used later to diff created/updated/deleted + const preExtractIndex = new Map(index.map((e) => [e.filename, e.summary])); + // Also snapshot scene content so we can detect content-only changes vs metadata-only changes + const preExtractContent = new Map(); + for (const e of index) { + try { + const raw = await fs.readFile(path.join(sceneBlocksDir, e.filename), "utf-8"); + const block = parseSceneBlock(raw, e.filename); + preExtractContent.set(e.filename, block.content); + } catch { /* non-fatal */ } + } + + // Phase 3: Build prompt + const promptStartMs = Date.now(); + const memoriesJson = JSON.stringify( + memories.map((m) => ({ + content: m.content, + created_at: m.created_at, + id: m.id ?? "", + })), + null, + 2, + ); + + const currentTimestamp = formatTimestamp(new Date()); + + const prompt = buildSceneExtractionPrompt({ + memoriesJson, + sceneSummaries: sceneSummaries || "(无已有场景)", + currentTimestamp, + sceneCountWarning, + existingSceneFiles, + }); + this.logger?.debug?.(`${TAG} extract() prompt built: ${prompt.length} chars (${Date.now() - promptStartMs}ms)`); + + // Phase 4: Run LLM agent (sandboxed to scene_blocks/) + let llmOutput = ""; + let llmDurationMs = 0; + try { + this.logger?.debug?.(`${TAG} extract() starting LLM runner (timeout=${this.timeoutMs}ms, maxTokens=model default)...`); + const runnerStartMs = Date.now(); + llmOutput = await this.runner.run({ + prompt, + taskId: `scene-extract-${Date.now()}`, + timeoutMs: this.timeoutMs, + // maxTokens omitted → core uses the resolved model's maxTokens from catalog + workspaceDir: sceneBlocksDir, + }) ?? ""; + llmDurationMs = Date.now() - runnerStartMs; + this.logger?.debug?.(`${TAG} extract() LLM runner completed: ${llmDurationMs}ms`); + } catch (err) { + const errMsg = err instanceof Error ? err.message : String(err); + const totalMs = Date.now() - extractStartMs; + this.logger?.error(`${TAG} extract() LLM runner failed after ${totalMs}ms: ${errMsg}`); + return { memoriesProcessed: 0, success: false, error: errMsg }; + } + + // Phase 5: Subsequent processing — safe cleanup of soft-deleted files + // + // Security: The LLM has no `exec` tool and cannot run shell commands. + // Instead, it "deletes" files by writing empty content (soft-delete). + // Here we detect and remove those empty files before syncing the index, + // so syncSceneIndex won't re-index stale empty entries. + const cleanupStartMs = Date.now(); + let cleanedCount = 0; + try { + const allFiles = (await fs.readdir(sceneBlocksDir)).filter((f) => f.endsWith(".md")); + for (const file of allFiles) { + const filePath = path.join(sceneBlocksDir, file); + const content = await fs.readFile(filePath, "utf-8"); + if (content.trim().length === 0) { + await fs.unlink(filePath); + cleanedCount++; + this.logger?.debug?.(`${TAG} extract() removed soft-deleted file: ${file}`); + } + } + } catch (cleanupErr) { + // Non-fatal — log and continue to index sync + this.logger?.warn(`${TAG} extract() soft-delete cleanup error: ${cleanupErr instanceof Error ? cleanupErr.message : String(cleanupErr)}`); + } + this.logger?.debug?.(`${TAG} extract() soft-delete cleanup: removed ${cleanedCount} empty files (${Date.now() - cleanupStartMs}ms)`); + + // Phase 6: Sync scene index (rebuilds from remaining non-empty files) + const syncStartMs = Date.now(); + await syncSceneIndex(this.dataDir); + this.logger?.debug?.(`${TAG} extract() scene index synced: ${Date.now() - syncStartMs}ms`); + + // Phase 7: Update persona.md navigation (GAP-4 fix) + const navStartMs = Date.now(); + try { + await this.updateSceneNavigation(); + this.logger?.debug?.(`${TAG} extract() persona.md navigation updated: ${Date.now() - navStartMs}ms`); + } catch (navErr) { + // Non-fatal — log and continue + this.logger?.warn(`${TAG} extract() failed to update persona navigation: ${navErr instanceof Error ? navErr.message : String(navErr)}`); + } + + // Phase 8: Parse LLM output for out-of-band persona update signal + if (llmOutput) { + const signal = parsePersonaUpdateSignal(llmOutput); + if (signal) { + await cpManager.setPersonaUpdateRequest(signal.reason); + this.logger?.debug?.(`${TAG} extract() persona update requested by LLM: ${signal.reason}`); + } + } + + const totalMs = Date.now() - extractStartMs; + this.logger?.info(`${TAG} extract() completed: ${memories.length} memories processed in ${totalMs}ms`); + + // ── l2_extraction metric ── + if (this.instanceId && this.logger) { + // Read updated scene index to report final state + diff against pre-extract snapshot + let resultScenes: Array<{ title: string; summary: string; content: string; status: "created" | "updated" }> = []; + let scenesCreated = 0; + let scenesUpdated = 0; + let scenesDeleted = 0; + try { + const finalIndex = await readSceneIndex(this.dataDir); + const postFilenames = new Set(); + for (const e of finalIndex) { + postFilenames.add(e.filename); + const oldSummary = preExtractIndex.get(e.filename); + // Read scene block content from disk + let content = ""; + try { + const blockPath = path.join(sceneBlocksDir, e.filename); + const raw = await fs.readFile(blockPath, "utf-8"); + const block = parseSceneBlock(raw, e.filename); + content = block.content; + } catch { /* file read failure is non-fatal */ } + + if (oldSummary === undefined) { + // New scene + scenesCreated++; + resultScenes.push({ + title: e.filename.replace(/\.md$/, ""), + summary: e.summary, + content, + status: "created", + }); + } else { + // Existing scene — check if content actually changed (not just metadata) + const oldContent = preExtractContent.get(e.filename) ?? ""; + if (content !== oldContent) { + scenesUpdated++; + resultScenes.push({ + title: e.filename.replace(/\.md$/, ""), + summary: e.summary, + content, + status: "updated", + }); + } + // If only metadata (summary/heat) changed but content is the same, skip + } + } + // Scenes in pre-extract but missing from post-extract = deleted + for (const [filename] of preExtractIndex) { + if (!postFilenames.has(filename)) { + scenesDeleted++; + } + } + } catch { /* non-fatal */ } + + report("l2_extraction", { + inputMemoryCount: memories.length, + resultSceneCount: resultScenes.length, + resultScenes, + scenesCreated, + scenesUpdated, + scenesDeleted, + llmDurationMs, + totalDurationMs: totalMs, + success: true, + error: null, + }); + } + + return { memoriesProcessed: memories.length, success: true }; + } + + /** + * Build human-readable scene summaries for the prompt, + * and collect the list of existing scene filenames (relative). + * + * Includes a capacity counter at the top (e.g. "当前场景总数：5 / 15") + * so the LLM can immediately see how close it is to the limit. + */ + private buildSceneSummaries( + index: SceneIndexEntry[], + ): { summaries: string; filenames: string[] } { + if (index.length === 0) return { summaries: "", filenames: [] }; + + const lines: string[] = []; + const filenames: string[] = []; + + // Inject capacity counter at the top — LLM sees this first + lines.push(`**当前场景总数：${index.length} / ${this.maxScenes}**`); + lines.push(""); + + for (const entry of index) { + filenames.push(entry.filename); + lines.push(`### ${entry.filename}`); + lines.push(`**热度**: ${entry.heat} | **更新**: ${entry.updated}`); + lines.push(`**summary**: ${entry.summary}`); + lines.push(""); + } + return { summaries: lines.join("\n"), filenames }; + } + + /** + * Update the scene navigation section at the end of persona.md. + * + * Reads the current scene index, generates the navigation block, then + * strips any existing navigation from persona.md and appends the new one. + * + * IMPORTANT: If the persona body is empty (PersonaGenerator hasn't run yet), + * we skip writing to avoid creating a persona.md that only contains the + * scene navigation. PersonaGenerator.generate() will write the full + * persona + navigation when it runs. + */ + private async updateSceneNavigation(): Promise { + const personaPath = path.join(this.dataDir, "persona.md"); + const index = await readSceneIndex(this.dataDir); + const nav = generateSceneNavigation(index); + + let existing = ""; + try { + existing = await fs.readFile(personaPath, "utf-8"); + } catch { + // No persona file yet — PersonaGenerator will create it with navigation. + // Don't write a navigation-only file. + this.logger?.debug?.(`${TAG} updateSceneNavigation() skipped: no persona file yet, waiting for PersonaGenerator`); + return; + } + + if (!existing.trim() && !nav) return; + + const stripped = stripSceneNavigation(existing).trimEnd(); + + // If the persona body is empty (only navigation existed), don't overwrite + // with a navigation-only file. Let PersonaGenerator handle full generation. + if (!stripped) { + this.logger?.debug?.(`${TAG} updateSceneNavigation() skipped: persona body is empty, waiting for PersonaGenerator`); + return; + } + + const updated = nav ? `${stripped}\n\n${nav}\n` : `${stripped}\n`; + + // persona.md is at dataDir root, no subdir needed + await fs.writeFile(personaPath, updated, "utf-8"); + } +} + +function formatTimestamp(d: Date): string { + return d.toISOString(); +} diff --git a/src/scene/scene-format.ts b/src/scene/scene-format.ts new file mode 100644 index 0000000..cb12628 --- /dev/null +++ b/src/scene/scene-format.ts @@ -0,0 +1,75 @@ +/** + * Scene Block file format: parse and format the META-delimited Markdown files. + */ + +export interface SceneBlockMeta { + created: string; + updated: string; + summary: string; + heat: number; +} + +export interface SceneBlock { + filename: string; + meta: SceneBlockMeta; + content: string; +} + +const META_START = "-----META-START-----"; +const META_END = "-----META-END-----"; + +/** + * Parse a Scene Block file into structured data. + */ +export function parseSceneBlock(raw: string, filename: string): SceneBlock { + const startIdx = raw.indexOf(META_START); + const endIdx = raw.indexOf(META_END); + + if (startIdx === -1 || endIdx === -1) { + // No META section — treat entire file as content + return { + filename, + meta: { created: "", updated: "", summary: "", heat: 0 }, + content: raw.trim(), + }; + } + + const metaBlock = raw.slice(startIdx + META_START.length, endIdx).trim(); + const content = raw.slice(endIdx + META_END.length).trim(); + + const meta: SceneBlockMeta = { + created: extractMetaField(metaBlock, "created"), + updated: extractMetaField(metaBlock, "updated"), + summary: extractMetaField(metaBlock, "summary"), + heat: parseInt(extractMetaField(metaBlock, "heat"), 10) || 0, + }; + + return { filename, meta, content }; +} + +/** + * Format a Scene Block back into file content. + */ +export function formatSceneBlock(meta: SceneBlockMeta, content: string): string { + return `${formatMeta(meta)}\n\n${content}`; +} + +/** + * Format the META section. + */ +export function formatMeta(meta: SceneBlockMeta): string { + return [ + META_START, + `created: ${meta.created}`, + `updated: ${meta.updated}`, + `summary: ${meta.summary}`, + `heat: ${meta.heat}`, + META_END, + ].join("\n"); +} + +function extractMetaField(metaBlock: string, field: string): string { + const re = new RegExp(`^${field}:\\s*(.*)$`, "m"); + const m = metaBlock.match(re); + return m ? m[1]!.trim() : ""; +} diff --git a/src/scene/scene-index.ts b/src/scene/scene-index.ts new file mode 100644 index 0000000..84f7d47 --- /dev/null +++ b/src/scene/scene-index.ts @@ -0,0 +1,96 @@ +/** + * Scene Index: maintains a JSON index of all scene blocks for quick lookup. + */ + +import fs from "node:fs/promises"; +import path from "node:path"; +import { parseSceneBlock } from "./scene-format.js"; + +export interface SceneIndexEntry { + filename: string; + summary: string; + heat: number; + created: string; + updated: string; +} + +/** + * Read the scene index from disk. + * + * The index is written exclusively by syncSceneIndex() (engineering side). + * The LLM is sandboxed to scene_blocks/ and cannot access this file. + */ +export async function readSceneIndex(dataDir: string): Promise { + const indexPath = path.join(dataDir, ".metadata", "scene_index.json"); + try { + const raw = await fs.readFile(indexPath, "utf-8"); + const parsed = JSON.parse(raw) as Array>; + if (!Array.isArray(parsed)) return []; + + const entries: SceneIndexEntry[] = []; + for (const item of parsed) { + if (!item || typeof item !== "object") continue; + + const filename = typeof item.filename === "string" ? item.filename : ""; + if (!filename) continue; + + entries.push({ + filename, + summary: typeof item.summary === "string" ? item.summary : "", + heat: typeof item.heat === "number" ? item.heat : 0, + created: typeof item.created === "string" ? item.created : "", + updated: typeof item.updated === "string" ? item.updated : "", + }); + } + return entries; + } catch { + return []; + } +} + +/** + * Write the scene index to disk. + */ +export async function writeSceneIndex( + dataDir: string, + entries: SceneIndexEntry[], +): Promise { + const indexPath = path.join(dataDir, ".metadata", "scene_index.json"); + await fs.mkdir(path.dirname(indexPath), { recursive: true }); + await fs.writeFile(indexPath, JSON.stringify(entries, null, 2), "utf-8"); +} + +/** + * Rebuild scene index by scanning all .md files in the scene_blocks directory. + */ +export async function syncSceneIndex(dataDir: string): Promise { + const blocksDir = path.join(dataDir, "scene_blocks"); + let files: string[]; + try { + files = (await fs.readdir(blocksDir)).filter((f) => f.endsWith(".md")); + } catch { + files = []; + } + + const entries: SceneIndexEntry[] = []; + for (const file of files) { + try { + const raw = await fs.readFile(path.join(blocksDir, file), "utf-8"); + const block = parseSceneBlock(raw, file); + entries.push({ + filename: file, + summary: block.meta.summary, + heat: block.meta.heat, + created: block.meta.created, + updated: block.meta.updated, + }); + } catch { + // File may have been deleted between readdir and readFile (e.g. by concurrent + // SceneExtractor soft-delete). Skip it and continue syncing the rest. + continue; + } + } + + await writeSceneIndex(dataDir, entries); + return entries; +} diff --git a/src/scene/scene-navigation.ts b/src/scene/scene-navigation.ts new file mode 100644 index 0000000..616b23f --- /dev/null +++ b/src/scene/scene-navigation.ts @@ -0,0 +1,57 @@ +/** + * Scene navigation: generates a summary navigation section appended to persona.md. + * + * The navigation includes file paths so the agent can use read_file to load + * scene details on demand (progressive disclosure). + */ + +import type { SceneIndexEntry } from "./scene-index.js"; + +const NAV_HEADER = "---\n## 🗺️ Scene Navigation (Scene Index)"; + +const NAV_FOOTER = `📌 使用说明： +- Path 即 scene block 的相对路径，可按需使用 read_file 读取完整内容 +- 热度：该场景被记忆命中的累计次数，越高越重要 +- Summary：场景的核心要点摘要`; + +/** + * Build a fire-emoji string based on heat value (visual priority cue for the agent). + */ +function heatEmoji(heat: number): string { + if (heat >= 1000) return " 🔥🔥🔥🔥🔥"; + if (heat >= 500) return " 🔥🔥🔥🔥"; + if (heat >= 200) return " 🔥🔥🔥"; + if (heat >= 100) return " 🔥🔥"; + if (heat >= 50) return " 🔥"; + return ""; +} + +/** + * Generate the scene navigation Markdown section. + * + * Output format mirrors the v1 Python version so the agent can identify file + * paths and invoke read_file for on-demand scene loading. + */ +export function generateSceneNavigation(entries: SceneIndexEntry[]): string { + if (entries.length === 0) return ""; + + const sorted = [...entries].sort((a, b) => b.heat - a.heat); + + const blocks = sorted.map((e) => { + const pathLine = `### Path: scene_blocks/${e.filename}`; + const heatLine = `**热度**: ${e.heat}${heatEmoji(e.heat)}${e.updated ? ` | **更新**: ${e.updated}` : ""}`; + const summaryLine = `Summary: ${e.summary}`; + return `${pathLine}\n${heatLine}\n${summaryLine}`; + }); + + return `${NAV_HEADER}\n*以下是当前场景记忆的索引，可根据需要 read_file 读取详细内容。*\n\n${blocks.join("\n\n")}\n\n${NAV_FOOTER}`; +} + +/** + * Strip the scene navigation section from persona content. + */ +export function stripSceneNavigation(personaContent: string): string { + const idx = personaContent.indexOf(NAV_HEADER); + if (idx === -1) return personaContent; + return personaContent.slice(0, idx).trimEnd(); +} diff --git a/src/store/embedding.ts b/src/store/embedding.ts new file mode 100644 index 0000000..531efb3 --- /dev/null +++ b/src/store/embedding.ts @@ -0,0 +1,598 @@ +/** + * Embedding Service: converts text to vector embeddings. + * + * Supports two providers: + * - "openai": OpenAI-compatible embedding APIs (OpenAI, Azure OpenAI, self-hosted) + * - "local": node-llama-cpp with embeddinggemma-300m GGUF model (fully offline) + * + * When no remote embedding is configured, automatically falls back to local provider. + * + * Design: + * - Single `embed()` for one text, `embedBatch()` for multiple. + * - `getDimensions()` returns configured vector dimensions. + * - Throws on failure; callers decide fallback strategy. + */ + +// ============================ +// Types +// ============================ + +export interface OpenAIEmbeddingConfig { + /** Provider identifier — any value other than "local" (e.g. "openai", "deepseek", "azure", "qclaw") */ + provider: string; + /** API base URL (required — must be specified by user, e.g. "https://api.openai.com/v1") */ + baseUrl: string; + /** API Key (required) */ + apiKey: string; + /** Model name (required — must be specified by user) */ + model: string; + /** Output dimensions (required — must match the chosen model) */ + dimensions: number; + /** Local proxy URL (only for provider="qclaw") — requests are forwarded through this proxy with Remote-URL header */ + proxyUrl?: string; + /** Max input text length in characters before truncation (default: 5000). */ + maxInputChars?: number; + /** Timeout per API call in milliseconds (default: 10000). */ + timeoutMs?: number; +} + +export interface LocalEmbeddingConfig { + provider: "local"; + /** Custom GGUF model path (default: embeddinggemma-300m from HuggingFace) */ + modelPath?: string; + /** Model cache directory (default: node-llama-cpp default cache) */ + modelCacheDir?: string; +} + +export type EmbeddingConfig = OpenAIEmbeddingConfig | LocalEmbeddingConfig; + +/** Identifies the embedding provider + model for change detection. */ +export interface EmbeddingProviderInfo { + /** Provider identifier (e.g. "local", "openai", "deepseek") */ + provider: string; + /** Model identifier (e.g. "embeddinggemma-300m", "text-embedding-3-large") */ + model: string; +} + +export interface EmbeddingService { + /** Get embedding for a single text */ + embed(text: string): Promise; + /** Get embeddings for multiple texts (batched API call) */ + embedBatch(texts: string[]): Promise; + /** Return the configured vector dimensions */ + getDimensions(): number; + /** Return provider + model identifiers for change detection */ + getProviderInfo(): EmbeddingProviderInfo; + /** + * Whether the service is ready to serve embed requests. + * For remote providers (OpenAI), always true (stateless HTTP). + * For local providers, true only after model download + load completes. + */ + isReady(): boolean; + /** + * Start background warmup (model download + load). + * For remote providers, this is a no-op. + * For local providers, triggers async initialization without blocking. + * Safe to call multiple times (idempotent). + */ + startWarmup(): void; + /** Optional: release resources (model memory, GPU, etc.) on shutdown */ + close?(): void | Promise; +} + +/** + * Error thrown when embed() / embedBatch() is called before the local + * embedding model has finished downloading and loading. + * Callers should catch this and fall back to keyword-only mode. + */ +export class EmbeddingNotReadyError extends Error { + constructor(message?: string) { + super(message ?? "Local embedding model is not ready yet (still downloading or loading)"); + this.name = "EmbeddingNotReadyError"; + } +} + +// ============================ +// Logger interface +// ============================ + +interface Logger { + debug?: (message: string) => void; + info: (message: string) => void; + warn: (message: string) => void; + error: (message: string) => void; +} + +const TAG = "[memory-tdai][embedding]"; + +// ============================ +// Local (node-llama-cpp) implementation +// ============================ + +/** Default model: Google's embeddinggemma-300m, quantized Q8_0 (~300MB) */ +const DEFAULT_LOCAL_MODEL = + "hf:ggml-org/embeddinggemma-300m-qat-q8_0-GGUF/embeddinggemma-300m-qat-Q8_0.gguf"; + +/** embeddinggemma-300m outputs 768-dimensional vectors */ +const LOCAL_DIMENSIONS = 768; + +/** + * embeddinggemma-300m has a 256-token context window. + * As a safe heuristic, we limit input to ~600 chars for CJK text + * (CJK characters typically tokenize to 1-2 tokens each, + * so 600 chars ≈ 200-400 tokens, keeping well within 256-token limit + * after accounting for special tokens). + * For Latin text, ~800 chars is a safe limit (~200 tokens). + * We use 512 chars as a conservative universal limit. + */ +const LOCAL_MAX_INPUT_CHARS = 512; + +/** + * Sanitize NaN/Inf values and L2-normalize the vector. + * Matches OpenClaw's own sanitizeAndNormalizeEmbedding(). + */ +function sanitizeAndNormalize(vec: number[] | Float32Array): Float32Array { + const arr = Array.from(vec).map((v) => (Number.isFinite(v) ? v : 0)); + const magnitude = Math.sqrt(arr.reduce((sum, v) => sum + v * v, 0)); + if (magnitude < 1e-10) { + return new Float32Array(arr); + } + return new Float32Array(arr.map((v) => v / magnitude)); +} + +/** + * Initialization state for LocalEmbeddingService. + * - "idle": not started yet + * - "initializing": model download / load is in progress (background) + * - "ready": model is loaded and ready to serve + * - "failed": initialization failed (will retry on next startWarmup) + */ +type LocalInitState = "idle" | "initializing" | "ready" | "failed"; + +export class LocalEmbeddingService implements EmbeddingService { + private readonly modelPath: string; + private readonly modelCacheDir?: string; + private readonly logger?: Logger; + + // Initialization state machine + private initState: LocalInitState = "idle"; + private initPromise: Promise | null = null; + private initError: Error | null = null; + private embeddingContext: { + getEmbeddingFor: (text: string) => Promise<{ vector: Float32Array | number[] }>; + } | null = null; + + constructor(config?: LocalEmbeddingConfig, logger?: Logger) { + this.modelPath = config?.modelPath?.trim() || DEFAULT_LOCAL_MODEL; + this.modelCacheDir = config?.modelCacheDir?.trim(); + this.logger = logger; + } + + getDimensions(): number { + return LOCAL_DIMENSIONS; + } + + getProviderInfo(): EmbeddingProviderInfo { + return { provider: "local", model: this.modelPath }; + } + + /** + * Whether the local model is fully loaded and ready to serve requests. + */ + isReady(): boolean { + return this.initState === "ready" && this.embeddingContext !== null; + } + + /** + * Start background warmup: download model (if needed) and load into memory. + * Does NOT block the caller — returns immediately. + * Safe to call multiple times (idempotent); re-triggers on "failed" state. + */ + startWarmup(): void { + if (this.initState === "initializing" || this.initState === "ready") { + return; // already in progress or done + } + this.logger?.info(`${TAG} Starting background warmup for local embedding model...`); + this.initState = "initializing"; + this.initError = null; + + this.initPromise = this._doInitialize() + .then(() => { + this.initState = "ready"; + this.logger?.info(`${TAG} Background warmup complete — local embedding ready`); + }) + .catch((err) => { + this.initState = "failed"; + this.initError = err instanceof Error ? err : new Error(String(err)); + this.logger?.error( + `${TAG} Background warmup failed: ${this.initError.message}. ` + + `embed() calls will throw EmbeddingNotReadyError until retried.`, + ); + }); + } + + /** + * Get embedding for a single text. + * @throws {EmbeddingNotReadyError} if model is not yet ready. + */ + async embed(text: string): Promise { + this.assertReady(); + const truncated = this.truncateInput(text); + const embedding = await this.embeddingContext!.getEmbeddingFor(truncated); + return sanitizeAndNormalize(embedding.vector); + } + + /** + * Get embeddings for multiple texts. + * @throws {EmbeddingNotReadyError} if model is not yet ready. + */ + async embedBatch(texts: string[]): Promise { + if (texts.length === 0) return []; + this.assertReady(); + + const results: Float32Array[] = []; + for (const text of texts) { + const truncated = this.truncateInput(text); + const embedding = await this.embeddingContext!.getEmbeddingFor(truncated); + results.push(sanitizeAndNormalize(embedding.vector)); + } + return results; + } + + /** + * Release the node-llama-cpp embedding context and model resources. + * Safe to call multiple times (idempotent). + */ + close(): void { + if (this.embeddingContext) { + try { + const ctx = this.embeddingContext as unknown as { dispose?: () => void }; + ctx.dispose?.(); + } catch { + // best-effort cleanup + } + this.embeddingContext = null; + this.initPromise = null; + this.initState = "idle"; + this.initError = null; + this.logger?.info(`${TAG} Local embedding resources released`); + } + } + + /** + * Assert the model is ready. Throws EmbeddingNotReadyError if not. + */ + private assertReady(): void { + if (this.initState === "ready" && this.embeddingContext) { + return; + } + if (this.initState === "failed") { + throw new EmbeddingNotReadyError( + `Local embedding model initialization failed: ${this.initError?.message ?? "unknown error"}. ` + + `Call startWarmup() to retry.`, + ); + } + if (this.initState === "initializing") { + throw new EmbeddingNotReadyError( + "Local embedding model is still loading (download/initialization in progress). Please try again later.", + ); + } + // "idle" — startWarmup() was never called + throw new EmbeddingNotReadyError( + "Local embedding model warmup has not been started. Call startWarmup() first.", + ); + } + + /** + * Truncate input text to stay within the model's context window. + * embeddinggemma-300m has a 256-token limit; we use a character-based + * heuristic (LOCAL_MAX_INPUT_CHARS) as a safe proxy. + */ + private truncateInput(text: string): string { + if (text.length <= LOCAL_MAX_INPUT_CHARS) return text; + this.logger?.debug?.( + `${TAG} Input truncated from ${text.length} to ${LOCAL_MAX_INPUT_CHARS} chars (model context limit)`, + ); + return text.slice(0, LOCAL_MAX_INPUT_CHARS); + } + + /** + * Internal: perform the actual model download + load. + * Called by startWarmup(), runs in background. + */ + private async _doInitialize(): Promise { + // Track partially-initialized resources for cleanup on failure + let model: { createEmbeddingContext: () => Promise; dispose?: () => void } | undefined; + try { + this.logger?.debug?.(`${TAG} Loading node-llama-cpp for local embedding...`); + + // Dynamic import — node-llama-cpp is a peer dependency of OpenClaw + const { getLlama, resolveModelFile, LlamaLogLevel } = await import("node-llama-cpp"); + + const llama = await getLlama({ logLevel: LlamaLogLevel.error }); + this.logger?.debug?.(`${TAG} Llama instance created`); + + const resolvedPath = await resolveModelFile( + this.modelPath, + this.modelCacheDir || undefined, + ); + this.logger?.debug?.(`${TAG} Model resolved: ${resolvedPath}`); + + model = await (llama as unknown as { loadModel: (opts: { modelPath: string }) => Promise }).loadModel({ modelPath: resolvedPath }); + this.logger?.debug?.(`${TAG} Model loaded, creating embedding context...`); + + this.embeddingContext = await model!.createEmbeddingContext() as typeof this.embeddingContext; + this.logger?.info(`${TAG} Local embedding ready (model=${this.modelPath}, dims=${LOCAL_DIMENSIONS})`); + } catch (err) { + // Clean up partially-initialized resources to prevent leaks + if (model?.dispose) { + try { model.dispose(); } catch { /* best-effort */ } + } + this.embeddingContext = null; + throw err; + } + } + + /** + * Wait for ongoing warmup to complete (used internally by tests). + * Returns immediately if already ready or idle. + */ + async waitForReady(): Promise { + if (this.initPromise) { + await this.initPromise; + } + } +} + +// ============================ +// OpenAI-compatible implementation +// ============================ + +/** Max texts per batch (OpenAI limit is 2048, we use a safe value) */ +const MAX_BATCH_SIZE = 256; + +/** Max retries for API calls */ +const MAX_RETRIES = 0; +/** Default timeout per API call in milliseconds */ +const DEFAULT_API_TIMEOUT_MS = 10_000; + +/** + * Custom error class for embedding API errors that carries HTTP status code. + * Used to distinguish non-retryable client errors (4xx except 429) from + * retryable server errors (5xx) and rate limits (429). + */ +class EmbeddingApiError extends Error { + readonly httpStatus: number; + constructor(message: string, httpStatus: number) { + super(message); + this.name = "EmbeddingApiError"; + this.httpStatus = httpStatus; + } + /** Returns true for 4xx errors that should NOT be retried (excluding 429). */ + isClientError(): boolean { + return this.httpStatus >= 400 && this.httpStatus < 500 && this.httpStatus !== 429; + } +} + +interface OpenAIEmbeddingResponse { + data: Array<{ + index: number; + embedding: number[]; + }>; + usage?: { + prompt_tokens: number; + total_tokens: number; + }; +} + +export class OpenAIEmbeddingService implements EmbeddingService { + private readonly baseUrl: string; + private readonly apiKey: string; + private readonly model: string; + private readonly dims: number; + private readonly providerName: string; + private readonly proxyUrl?: string; + private readonly maxInputChars?: number; + private readonly timeoutMs: number; + private readonly logger?: Logger; + + constructor(config: OpenAIEmbeddingConfig, logger?: Logger) { + if (!config.apiKey) { + throw new Error("EmbeddingService: apiKey is required for remote provider"); + } + if (!config.baseUrl) { + throw new Error("EmbeddingService: baseUrl is required for remote provider"); + } + if (!config.model) { + throw new Error("EmbeddingService: model is required for remote provider"); + } + if (!config.dimensions || config.dimensions <= 0) { + throw new Error("EmbeddingService: dimensions is required for remote provider (must be a positive integer)"); + } + this.baseUrl = config.baseUrl.replace(/\/+$/, ""); + this.apiKey = config.apiKey; + this.model = config.model; + this.dims = config.dimensions; + this.providerName = config.provider || "openai"; + this.proxyUrl = config.proxyUrl?.trim() || undefined; + this.maxInputChars = config.maxInputChars && config.maxInputChars > 0 ? config.maxInputChars : undefined; + this.timeoutMs = config.timeoutMs && config.timeoutMs > 0 ? config.timeoutMs : DEFAULT_API_TIMEOUT_MS; + this.logger = logger; + } + + getDimensions(): number { + return this.dims; + } + + getProviderInfo(): EmbeddingProviderInfo { + return { provider: this.providerName, model: this.model }; + } + + /** Remote embedding is always ready (stateless HTTP). */ + isReady(): boolean { + return true; + } + + /** No-op for remote embedding (no local model to warm up). */ + startWarmup(): void { + // nothing to do — remote API is stateless + } + + async embed(text: string): Promise { + const [result] = await this.embedBatch([text]); + return result; + } + + async embedBatch(texts: string[]): Promise { + if (texts.length === 0) return []; + + // Truncate texts exceeding maxInputChars limit + const processedTexts = this.maxInputChars + ? texts.map((t) => this.truncateInput(t)) + : texts; + + // Split into sub-batches if needed + if (processedTexts.length > MAX_BATCH_SIZE) { + const results: Float32Array[] = []; + for (let i = 0; i < processedTexts.length; i += MAX_BATCH_SIZE) { + const chunk = processedTexts.slice(i, i + MAX_BATCH_SIZE); + const chunkResults = await this._callApi(chunk); + results.push(...chunkResults); + } + return results; + } + + return this._callApi(processedTexts); + } + + /** + * Truncate input text to stay within the configured maxInputChars limit. + * Logs a warning when truncation occurs. + */ + private truncateInput(text: string): string { + if (!this.maxInputChars || text.length <= this.maxInputChars) return text; + this.logger?.warn?.( + `${TAG} Input truncated from ${text.length} to ${this.maxInputChars} chars (maxInputChars limit)`, + ); + return text.slice(0, this.maxInputChars); + } + + private async _callApi(texts: string[]): Promise { + const body: Record = { + input: texts, + model: this.model, + dimensions: this.dims, + }; + + // Determine fetch URL and headers based on proxy mode + const useProxy = this.providerName === "qclaw" && !!this.proxyUrl; + const fetchUrl = useProxy ? this.proxyUrl! : `${this.baseUrl}/embeddings`; + const headers: Record = { + "Content-Type": "application/json", + Authorization: `Bearer ${this.apiKey}`, + }; + if (useProxy) { + headers["Remote-URL"] = `${this.baseUrl}/embeddings`; + this.logger?.debug?.( + `${TAG} [qclaw-proxy] Forwarding embedding request via proxy: ${fetchUrl}, Remote-URL: ${headers["Remote-URL"]}`, + ); + } + + // Retry loop with timeout + let lastError: Error | undefined; + for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) { + try { + const controller = new AbortController(); + const timeoutId = setTimeout(() => controller.abort(), this.timeoutMs); + + try { + const resp = await fetch(fetchUrl, { + method: "POST", + headers, + body: JSON.stringify(body), + signal: controller.signal, + }); + + if (!resp.ok) { + const errBody = await resp.text().catch(() => "(unable to read body)"); + const err = new EmbeddingApiError( + `Embedding API error: HTTP ${resp.status} ${resp.statusText} — ${errBody.slice(0, 500)}`, + resp.status, + ); + // Don't retry on 4xx client errors (except 429 rate limit) + if (resp.status >= 400 && resp.status < 500 && resp.status !== 429) { + throw err; + } + lastError = err; + continue; + } + + const json = (await resp.json()) as OpenAIEmbeddingResponse; + + if (!json.data || !Array.isArray(json.data)) { + throw new Error("Embedding API returned unexpected format: missing 'data' array"); + } + + // Sort by index to ensure correct order, then sanitize+normalize for consistency with local provider + const sorted = [...json.data].sort((a, b) => a.index - b.index); + return sorted.map((d) => sanitizeAndNormalize(d.embedding)); + } finally { + clearTimeout(timeoutId); + } + } catch (err) { + // Non-retryable errors (4xx client errors) — rethrow immediately + if (err instanceof EmbeddingApiError && err.isClientError()) { + throw err; + } + lastError = err instanceof Error ? err : new Error(String(err)); + // AbortError = timeout, retry + if (attempt < MAX_RETRIES) { + // Exponential backoff: 500ms, 1000ms + const delay = 500 * (attempt + 1); + await new Promise((r) => setTimeout(r, delay)); + } + } + } + + throw lastError ?? new Error("Embedding API call failed after retries"); + } +} + +// ============================ +// Factory +// ============================ + +/** + * Create an EmbeddingService from config. + * + * Strategy: + * - If config has provider != "local" with valid apiKey, model, and dimensions → use remote OpenAI-compatible embedding + * - If config has provider="local" → use node-llama-cpp local embedding + * - If config is undefined or missing required fields → fall back to local embedding + * + * NOTE: For local providers, `startWarmup()` is NOT called here. + * The caller is responsible for calling `startWarmup()` at the right time + * (e.g. on first conversation) to avoid triggering model download during + * short-lived CLI commands like `gateway stop` or `agents list`. + */ +export function createEmbeddingService( + config: EmbeddingConfig | undefined, + logger?: Logger, +): EmbeddingService { + // Remote OpenAI-compatible provider: any provider value other than "local" + if (config && config.provider !== "local" && "apiKey" in config && config.apiKey) { + logger?.info(`${TAG} Using remote embedding (provider=${config.provider}, model=${config.model})`); + return new OpenAIEmbeddingService(config as OpenAIEmbeddingConfig, logger); + } + + // Explicit local config + if (config && config.provider === "local") { + const localConfig = config as LocalEmbeddingConfig; + logger?.info(`${TAG} Using local embedding (node-llama-cpp, model=${localConfig.modelPath ?? DEFAULT_LOCAL_MODEL})`); + return new LocalEmbeddingService(localConfig, logger); + } + + // Fallback: no config or empty apiKey → use local + logger?.info(`${TAG} No remote embedding configured, falling back to local embedding (node-llama-cpp)`); + return new LocalEmbeddingService(undefined, logger); +} diff --git a/src/store/vector-store.ts b/src/store/vector-store.ts new file mode 100644 index 0000000..4696f10 --- /dev/null +++ b/src/store/vector-store.ts @@ -0,0 +1,2225 @@ +/** + * VectorStore: SQLite-based vector storage using sqlite-vec extension. + * + * Manages two layers of vector-indexed data in a single SQLite database: + * + * **L1 (structured memories):** + * 1. `l1_records` — relational metadata table (content, type, priority, scene, timestamps) + * 2. `l1_vec` — vec0 virtual table for cosine similarity search + * + * **L0 (raw conversations):** + * 3. `l0_conversations` — relational metadata table (session_key, role, message text, timestamps) + * 4. `l0_vec` — vec0 virtual table for cosine similarity search on individual messages + * + * Dependencies: Node.js built-in `node:sqlite` (Node 22+) + `sqlite-vec` (from root workspace). + * + * Design: + * - All operations are synchronous (DatabaseSync API). + * - Writes use manual BEGIN/COMMIT transactions for atomicity (metadata + vector). + * - vec0 virtual table does NOT support ON CONFLICT, so upsert = delete + insert. + * - Thread-safe via WAL mode. + */ + +import { createRequire } from "node:module"; +import type { DatabaseSync, StatementSync } from "node:sqlite"; +import type { MemoryRecord } from "../record/l1-writer.js"; +import type { EmbeddingProviderInfo } from "./embedding.js"; + +// ============================ +// Types +// ============================ + +export interface VectorSearchResult { + record_id: string; + content: string; + type: string; + priority: number; + scene_name: string; + /** Cosine similarity score (1.0 - cosine_distance) */ + score: number; + timestamp_str: string; + timestamp_start: string; + timestamp_end: string; + session_key: string; + session_id: string; + /** Raw metadata JSON string (e.g., contains activity_start_time / activity_end_time for episodic) */ + metadata_json: string; +} + +/** L0 single-message record for vector indexing. */ +export interface L0VectorRecord { + /** Unique ID for this message (e.g., "l0___") */ + id: string; + /** Session key this message belongs to */ + sessionKey: string; + /** Session ID (single conversation instance) this message belongs to */ + sessionId: string; + /** Role of the message sender: "user" or "assistant" */ + role: string; + /** Text content of this single message */ + messageText: string; + /** ISO timestamp when this message was recorded */ + recordedAt: string; + /** Original message timestamp (epoch ms) — used by L1 cursor for incremental filtering */ + timestamp: number; +} + +/** L0 single-message vector search result. */ +export interface L0VectorSearchResult { + record_id: string; + session_key: string; + session_id: string; + role: string; + message_text: string; + /** Cosine similarity score (1.0 - cosine_distance) */ + score: number; + recorded_at: string; + /** Original message timestamp (epoch ms) */ + timestamp: number; +} + +/** Raw row returned by L1 record queries (column names match SQLite schema). */ +export interface L1RecordRow { + record_id: string; + content: string; + type: string; + priority: number; + scene_name: string; + session_key: string; + session_id: string; + timestamp_str: string; + timestamp_start: string; + timestamp_end: string; + created_time: string; + updated_time: string; + metadata_json: string; +} + +/** Filter options for querying L1 records from SQLite. */ +export interface L1QueryFilter { + /** If provided, only return records for this session key (conversation channel). */ + sessionKey?: string; + /** If provided, only return records for this session ID (single conversation instance). */ + sessionId?: string; + /** If provided, only return records with updated_time strictly after this ISO 8601 UTC timestamp. */ + updatedAfter?: string; +} + +interface Logger { + debug?: (message: string) => void; + info: (message: string) => void; + warn: (message: string) => void; + error: (message: string) => void; +} + +const TAG = "[memory-tdai][vector-store]"; + +/** Persisted metadata about the embedding provider used to generate stored vectors. */ +interface EmbeddingMeta { + provider: string; + model: string; + dimensions: number; +} + +/** Result of VectorStore.init() — indicates whether a re-embed is needed. */ +export interface VectorStoreInitResult { + /** + * `true` if the embedding provider/model/dimensions changed since + * the vectors were last written. Callers should re-embed all texts + * (via `reindexAll()`) after receiving this flag. + */ + needsReindex: boolean; + /** Human-readable reason (for logging). */ + reason?: string; +} + +// Use createRequire to load the experimental node:sqlite module +const require = createRequire(import.meta.url); + +function requireNodeSqlite(): typeof import("node:sqlite") { + return require("node:sqlite") as typeof import("node:sqlite"); +} + +// ============================ +// FTS5 helpers (adapted from openclaw core hybrid.ts) +// ============================ + +// ── Chinese word segmentation (jieba) ── +// Lazy-loaded singleton: initialised on first call to `buildFtsQuery`. +// If @node-rs/jieba is unavailable, falls back to Unicode-regex splitting. + +interface JiebaInstance { + cutForSearch(text: string, hmm: boolean): string[]; +} + +let _jieba: JiebaInstance | null | undefined; // undefined = not yet tried + +function getJieba(): JiebaInstance | null { + if (_jieba !== undefined) return _jieba; + try { + // eslint-disable-next-line @typescript-eslint/no-require-imports + const { Jieba } = require("@node-rs/jieba"); + // eslint-disable-next-line @typescript-eslint/no-require-imports + const { dict } = require("@node-rs/jieba/dict"); + _jieba = Jieba.withDict(dict) as JiebaInstance; + } catch { + _jieba = null; // mark as unavailable — won't retry + } + return _jieba; +} + +/** + * Common Chinese stop-words that add noise to FTS5 queries. + * Kept small on purpose — only high-frequency function words. + */ +const ZH_STOP_WORDS = new Set([ + "的", "了", "在", "是", "我", "有", "和", "就", "不", "人", "都", "一", + "一个", "上", "也", "很", "到", "说", "要", "去", "你", "会", "着", + "没有", "看", "好", "自己", "这", "他", "她", "它", "们", "那", + "吗", "吧", "呢", "啊", "呀", "哦", "嗯", +]); + +/** + * Build an FTS5 MATCH query from raw text. + * + * When `@node-rs/jieba` is available, uses jieba's search-engine mode + * (`cutForSearch`) for accurate Chinese word segmentation, producing + * much better recall than the previous regex-only approach. + * + * Falls back to Unicode-regex splitting (`/[\p{L}\p{N}_]+/gu`) if + * jieba is not installed. + * + * Tokens are OR-joined as quoted FTS5 phrase terms so that a document + * matching *any* token is returned. BM25 naturally ranks documents that + * match more tokens higher, so precision is preserved while recall is + * significantly improved — especially for longer queries and when running + * in FTS-only fallback mode (no embedding available). + * + * Example (with jieba): + * "用户喜欢编程和TypeScript" → '"用户" OR "喜欢" OR "编程" OR "TypeScript"' + * Example (fallback): + * "旅行计划 API" → '"旅行计划" OR "API"' + */ +export function buildFtsQuery(raw: string): string | null { + const jieba = getJieba(); + + let tokens: string[]; + if (jieba) { + // jieba cutForSearch: splits long words further for better recall + // e.g. "北京烤鸭" → ["北京", "烤鸭", "北京烤鸭"] + tokens = jieba + .cutForSearch(raw, true) + .map((t) => t.trim()) + .filter((t) => { + if (!t) return false; + // Remove pure whitespace / punctuation tokens + if (!/[\p{L}\p{N}]/u.test(t)) return false; + // Remove common Chinese stop-words to reduce noise + if (ZH_STOP_WORDS.has(t)) return false; + return true; + }); + // Deduplicate (cutForSearch may produce duplicates for sub-words) + tokens = [...new Set(tokens)]; + } else { + // Fallback: simple Unicode regex split + tokens = + raw + .match(/[\p{L}\p{N}_]+/gu) + ?.map((t) => t.trim()) + .filter(Boolean) ?? []; + } + + if (tokens.length === 0) return null; + const quoted = tokens.map((t) => `"${t.replaceAll('"', "")}"`); + return quoted.join(" OR "); +} + +/** + * Tokenize text for FTS5 indexing (write-side). + * + * Uses jieba `cutForSearch()` (search-engine mode) to segment Chinese text, + * then joins tokens with spaces. The resulting string is stored in the FTS5 + * `content` column so that `unicode61` tokenizer can split it into meaningful + * words — including both full words and their sub-words. + * + * Using `cutForSearch` (instead of `cut`) ensures that the index contains + * the same sub-word tokens that `buildFtsQuery()` produces on the query side. + * For example, "人工智能" is indexed as "人工 智能 人工智能", so queries for + * either the full term or sub-words will match. + * + * Falls back to the original text if jieba is unavailable. + * + * Example (with jieba): + * "用户五月去日本旅行" → "用户 五月 去 日本 旅行" + * "人工智能的分支" → "人工 智能 人工智能 的 分支" + * Example (fallback): + * "用户五月去日本旅行" → "用户五月去日本旅行" (unchanged) + */ +export function tokenizeForFts(raw: string): string { + const jieba = getJieba(); + if (!jieba) return raw; + + // Use `cutForSearch` (search-engine mode) for indexing — it produces both + // full words AND their sub-word components. This ensures that query-side + // tokens (also produced by `cutForSearch` in `buildFtsQuery`) will always + // find a match in the index. + const tokens = jieba.cutForSearch(raw, true); + + // Join with spaces so `unicode61` tokenizer can split them. + // Punctuation tokens are kept — unicode61 treats them as separators anyway. + return tokens.join(" "); +} + +/** + * Reset jieba state so next call to `buildFtsQuery` re-initialises. + * Exported for testing only. + * @internal + */ +export function _resetJiebaForTest(): void { + _jieba = undefined; +} + +/** + * Override jieba instance (or set to `null` to force fallback). + * Exported for testing only. + * @internal + */ +export function _setJiebaForTest(instance: JiebaInstance | null): void { + _jieba = instance; +} + +/** + * Convert a BM25 rank (negative = more relevant) to a 0–1 score. + * Mirrors the formula in openclaw core `hybrid.ts`. + */ +export function bm25RankToScore(rank: number): number { + if (!Number.isFinite(rank)) return 1 / (1 + 999); + if (rank < 0) { + const relevance = -rank; + return relevance / (1 + relevance); + } + return 1 / (1 + rank); +} + +/** FTS5 search result for L1 records. */ +export interface FtsSearchResult { + record_id: string; + content: string; + type: string; + priority: number; + scene_name: string; + /** BM25-derived score (0–1, higher is better) */ + score: number; + timestamp_str: string; + timestamp_start: string; + timestamp_end: string; + session_key: string; + session_id: string; + metadata_json: string; +} + +/** FTS5 search result for L0 records. */ +export interface L0FtsSearchResult { + record_id: string; + session_key: string; + session_id: string; + role: string; + message_text: string; + /** BM25-derived score (0–1, higher is better) */ + score: number; + recorded_at: string; + timestamp: number; +} + +// ============================ +// VectorStore class +// ============================ + +export class VectorStore { + private db: DatabaseSync; + private readonly dimensions: number; + private readonly logger?: Logger; + + /** + * When `true`, the store is in a degraded state (e.g. sqlite-vec failed to + * load, or init() encountered an unrecoverable error). All public methods + * become safe no-ops so the plugin never blocks the main OpenClaw flow. + */ + private degraded = false; + + /** Tracks whether close() has been called to prevent double-close errors. */ + private closed = false; + + /** + * `true` when vec0 virtual tables (l1_vec / l0_vec) have been created and + * their prepared statements are ready. When `dimensions === 0` (i.e. + * provider="none"), vec0 tables are deferred and this stays `false`. + */ + private vecTablesReady = false; + + // Prepared statements — L1 (initialized in init()) + private stmtUpsertMeta!: StatementSync; + private stmtDeleteVec?: StatementSync; // optional — only set when vecTablesReady + private stmtInsertVec?: StatementSync; // optional — only set when vecTablesReady + private stmtDeleteMeta!: StatementSync; + private stmtGetMeta!: StatementSync; + private stmtSearchVec?: StatementSync; // optional — only set when vecTablesReady + private stmtQueryBySessionId!: StatementSync; + private stmtQueryBySessionIdSince!: StatementSync; + private stmtQueryBySessionKey!: StatementSync; + private stmtQueryBySessionKeySince!: StatementSync; + private stmtQueryAll!: StatementSync; + private stmtQueryAllSince!: StatementSync; + + // Prepared statements — L0 (initialized in init()) + private stmtL0UpsertMeta!: StatementSync; + private stmtL0DeleteVec?: StatementSync; // optional — only set when vecTablesReady + private stmtL0InsertVec?: StatementSync; // optional — only set when vecTablesReady + private stmtL0DeleteMeta!: StatementSync; + private stmtL0GetMeta!: StatementSync; + private stmtL0SearchVec?: StatementSync; // optional — only set when vecTablesReady + /** L0 query for L1 runner: all messages for a session key */ + private stmtL0QueryAll!: StatementSync; + /** L0 query for L1 runner: messages after a timestamp cursor */ + private stmtL0QueryAfter!: StatementSync; + + // FTS5 tables availability flag (created best-effort — may be false if fts5 is not compiled in) + private ftsAvailable = false; + + // Prepared statements — FTS5 L1 (initialized in init()) + private stmtL1FtsInsert!: StatementSync; + private stmtL1FtsDelete!: StatementSync; + private stmtL1FtsSearch!: StatementSync; + + // Prepared statements — FTS5 L0 (initialized in init()) + private stmtL0FtsInsert!: StatementSync; + private stmtL0FtsDelete!: StatementSync; + private stmtL0FtsSearch!: StatementSync; + + /** + * Create a VectorStore instance. + * + * Note: After construction, you MUST call `init()` to load the sqlite-vec + * extension and create the schema. + */ + constructor(dbPath: string, dimensions: number, logger?: Logger) { + this.dimensions = dimensions; + this.logger = logger; + + // Open database with extension support enabled + const { DatabaseSync: DbSync } = requireNodeSqlite(); + this.db = new DbSync(dbPath, { allowExtension: true }); + + // Set busy timeout so concurrent processes retry instead of failing with SQLITE_BUSY + this.db.exec("PRAGMA busy_timeout = 5000"); + + // Enable WAL mode for better concurrent read performance + this.db.exec("PRAGMA journal_mode = WAL"); + + // Cap page cache at 64 MB + this.db.exec("PRAGMA cache_size = -65536"); + + // Cap memory-mapped I/O at 128 MB to bound RSS growth + this.db.exec("PRAGMA mmap_size = 134217728"); + + // Auto-checkpoint WAL every 1000 pages (~4 MB) to keep WAL file compact + this.db.exec("PRAGMA wal_autocheckpoint = 1000"); + } + + /** + * Whether the store is in degraded mode (e.g. sqlite-vec failed to load). + * When degraded, all write/search operations become safe no-ops. + */ + isDegraded(): boolean { + return this.degraded; + } + + + /** + * Load sqlite-vec extension and initialize database schema. + * Must be called once after construction. + * + * @param providerInfo Current embedding provider info. When provided, + * the store compares it against the persisted metadata. If the provider, + * model, or dimensions changed, the vector tables are dropped and + * re-created with the new dimensions, and `needsReindex: true` is returned + * so the caller can schedule a full re-embed. + */ + init(providerInfo?: EmbeddingProviderInfo): VectorStoreInitResult { + // Load sqlite-vec extension (same approach as root project's sqlite-vec.ts) + try { + // eslint-disable-next-line @typescript-eslint/no-require-imports + const sqliteVec = require("sqlite-vec"); + this.db.enableLoadExtension(true); + sqliteVec.load(this.db); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + this.logger?.error( + `${TAG} Failed to load sqlite-vec extension: ${message}. ` + + `VectorStore entering degraded mode — all operations will be no-ops.`, + ); + this.degraded = true; + return { needsReindex: false, reason: `sqlite-vec load failed: ${message}` }; + } + + // ── Schema creation & prepared statements ────────────────────────────── + // Wrapped in try-catch: if anything fails during schema init (e.g. the DB + // is corrupted, disk full, etc.), we degrade gracefully instead of crashing. + try { + return this.initSchema(providerInfo); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + this.logger?.error( + `${TAG} Schema initialization failed: ${message}. ` + + `VectorStore entering degraded mode.`, + ); + this.degraded = true; + return { needsReindex: false, reason: `schema init failed: ${message}` }; + } + } + + /** + * Internal schema initialization — separated from init() so we can + * catch errors at the top level and degrade gracefully. + */ + private initSchema(providerInfo?: EmbeddingProviderInfo): VectorStoreInitResult { + // Tracks which provider/model/dimensions were used to generate vectors. + this.db.exec(` + CREATE TABLE IF NOT EXISTS embedding_meta ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + ) + `); + + // Detect whether re-index is needed + let needsReindex = false; + let reindexReason: string | undefined; + + const savedMeta = this.readEmbeddingMeta(); + + if (providerInfo) { + if (savedMeta) { + const providerChanged = savedMeta.provider !== providerInfo.provider; + const modelChanged = savedMeta.model !== providerInfo.model; + const dimsChanged = savedMeta.dimensions !== this.dimensions; + + if (providerChanged || modelChanged || dimsChanged) { + const reasons: string[] = []; + if (providerChanged) reasons.push(`provider: ${savedMeta.provider} → ${providerInfo.provider}`); + if (modelChanged) reasons.push(`model: ${savedMeta.model} → ${providerInfo.model}`); + if (dimsChanged) reasons.push(`dimensions: ${savedMeta.dimensions} → ${this.dimensions}`); + reindexReason = reasons.join(", "); + + this.logger?.info( + `${TAG} Embedding config changed (${reindexReason}). ` + + `Dropping vector tables for rebuild...`, + ); + + // Drop and re-create vector tables with new dimensions + this.dropVectorTables(); + needsReindex = true; + } + } else { + // No saved meta — first run or legacy DB without meta table. + // Two cases require dropping vector tables: + // 1. Existing data created without meta tracking (legacy DB) — need re-embed + // 2. vec0 tables exist with wrong dimensions (e.g. previously created with + // provider="none" placeholder 768D, now switching to a real provider + // with different dimensions) — must rebuild even if data tables are empty + const l1Count = this.tableRowCount("l1_records"); + const l0Count = this.tableRowCount("l0_conversations"); + const existingVecDims = this.getVecTableDimensions(); + + if (l1Count > 0 || l0Count > 0) { + this.logger?.info( + `${TAG} No embedding_meta found but existing data exists ` + + `(L1=${l1Count}, L0=${l0Count}). Dropping vector tables for safety...`, + ); + this.dropVectorTables(); + needsReindex = true; + reindexReason = "legacy DB without embedding_meta — cannot verify vector compatibility"; + } else if (existingVecDims !== null && existingVecDims !== this.dimensions) { + // vec0 tables exist (from a previous provider="none" placeholder or + // different config) but with mismatched dimensions. Drop them so they + // get re-created with the correct dimensions below. + this.logger?.info( + `${TAG} vec0 table dimension mismatch (existing=${existingVecDims}, ` + + `required=${this.dimensions}). Dropping vector tables for rebuild...`, + ); + this.dropVectorTables(); + // No needsReindex — there's no data to re-embed + } + } + } + + // ── L1 schema ────────────────────────────────── + + // Metadata table + this.db.exec(` + CREATE TABLE IF NOT EXISTS l1_records ( + record_id TEXT PRIMARY KEY, + content TEXT NOT NULL, + type TEXT DEFAULT '', + priority INTEGER DEFAULT 50, + scene_name TEXT DEFAULT '', + session_key TEXT DEFAULT '', + session_id TEXT DEFAULT '', + timestamp_str TEXT DEFAULT '', + timestamp_start TEXT DEFAULT '', + timestamp_end TEXT DEFAULT '', + created_time TEXT DEFAULT '', + updated_time TEXT DEFAULT '', + metadata_json TEXT DEFAULT '{}' + ) + `); + + // Indexes for common queries + this.db.exec("CREATE INDEX IF NOT EXISTS idx_l1_type ON l1_records(type)"); + this.db.exec("CREATE INDEX IF NOT EXISTS idx_l1_session_key ON l1_records(session_key)"); + this.db.exec("CREATE INDEX IF NOT EXISTS idx_l1_session_id ON l1_records(session_id)"); + this.db.exec("CREATE INDEX IF NOT EXISTS idx_l1_scene ON l1_records(scene_name)"); + this.db.exec("CREATE INDEX IF NOT EXISTS idx_l1_ts_start ON l1_records(timestamp_start)"); + this.db.exec("CREATE INDEX IF NOT EXISTS idx_l1_ts_end ON l1_records(timestamp_end)"); + // Composite index: session_id exact match + updated_time range scan (for incremental L2 queries) + this.db.exec("CREATE INDEX IF NOT EXISTS idx_l1_session_updated ON l1_records(session_id, updated_time)"); + // Composite index: session_key exact match + updated_time range scan (for pipeline cursor queries) + this.db.exec("CREATE INDEX IF NOT EXISTS idx_l1_sessionkey_updated ON l1_records(session_key, updated_time)"); + + // Vector virtual table (cosine distance) — only created when dimensions > 0. + // When provider="none", dimensions=0 and vec0 tables are deferred until a + // real embedding provider is configured. + if (this.dimensions > 0) { + this.db.exec(` + CREATE VIRTUAL TABLE IF NOT EXISTS l1_vec USING vec0( + record_id TEXT PRIMARY KEY, + embedding float[${this.dimensions}] distance_metric=cosine, + updated_time TEXT DEFAULT '' + ) + `); + } + + // Prepare statements for reuse + this.stmtUpsertMeta = this.db.prepare(` + INSERT INTO l1_records ( + record_id, content, type, priority, scene_name, session_key, session_id, + timestamp_str, timestamp_start, timestamp_end, + created_time, updated_time, metadata_json + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(record_id) DO UPDATE SET + content=excluded.content, + type=excluded.type, + priority=excluded.priority, + scene_name=excluded.scene_name, + timestamp_str=excluded.timestamp_str, + timestamp_start=excluded.timestamp_start, + timestamp_end=excluded.timestamp_end, + updated_time=excluded.updated_time, + metadata_json=excluded.metadata_json + `); + + if (this.dimensions > 0) { + this.stmtDeleteVec = this.db.prepare("DELETE FROM l1_vec WHERE record_id = ?"); + this.stmtInsertVec = this.db.prepare("INSERT INTO l1_vec (record_id, embedding, updated_time) VALUES (?, ?, ?)"); + } + this.stmtDeleteMeta = this.db.prepare("DELETE FROM l1_records WHERE record_id = ?"); + + this.stmtGetMeta = this.db.prepare(` + SELECT content, type, priority, scene_name, session_key, session_id, + timestamp_str, timestamp_start, timestamp_end, metadata_json + FROM l1_records WHERE record_id = ? + `); + + if (this.dimensions > 0) { + this.stmtSearchVec = this.db.prepare(` + SELECT record_id, distance + FROM l1_vec + WHERE embedding MATCH ? + AND k = ? + ORDER BY distance + `); + } + + // ── L0 schema ────────────────────────────────── + + // L0 metadata table: stores individual messages for vector search + this.db.exec(` + CREATE TABLE IF NOT EXISTS l0_conversations ( + record_id TEXT PRIMARY KEY, + session_key TEXT NOT NULL, + session_id TEXT DEFAULT '', + role TEXT NOT NULL DEFAULT '', + message_text TEXT NOT NULL, + recorded_at TEXT DEFAULT '', + timestamp INTEGER DEFAULT 0 + ) + `); + + // Migration: add timestamp column if missing (existing DBs pre-v3.x) + try { + this.db.exec("ALTER TABLE l0_conversations ADD COLUMN timestamp INTEGER DEFAULT 0"); + this.logger?.info(`${TAG} Migrated l0_conversations: added timestamp column`); + } catch { + // Column already exists — expected on non-first run + } + + // Indexes for L0 queries + this.db.exec("CREATE INDEX IF NOT EXISTS idx_l0_session ON l0_conversations(session_key)"); + this.db.exec("CREATE INDEX IF NOT EXISTS idx_l0_session_id ON l0_conversations(session_id)"); + this.db.exec("CREATE INDEX IF NOT EXISTS idx_l0_recorded ON l0_conversations(recorded_at)"); + this.db.exec("CREATE INDEX IF NOT EXISTS idx_l0_timestamp ON l0_conversations(timestamp)"); + + // L0 vector virtual table (cosine distance, same dimensions as L1) — deferred when dimensions=0 + if (this.dimensions > 0) { + this.db.exec(` + CREATE VIRTUAL TABLE IF NOT EXISTS l0_vec USING vec0( + record_id TEXT PRIMARY KEY, + embedding float[${this.dimensions}] distance_metric=cosine, + recorded_at TEXT DEFAULT '' + ) + `); + } + + // L0 prepared statements + this.stmtL0UpsertMeta = this.db.prepare(` + INSERT INTO l0_conversations ( + record_id, session_key, session_id, role, message_text, recorded_at, timestamp + ) VALUES (?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(record_id) DO UPDATE SET + message_text=excluded.message_text, + recorded_at=excluded.recorded_at, + timestamp=excluded.timestamp + `); + + if (this.dimensions > 0) { + this.stmtL0DeleteVec = this.db.prepare("DELETE FROM l0_vec WHERE record_id = ?"); + this.stmtL0InsertVec = this.db.prepare("INSERT INTO l0_vec (record_id, embedding, recorded_at) VALUES (?, ?, ?)"); + } + this.stmtL0DeleteMeta = this.db.prepare("DELETE FROM l0_conversations WHERE record_id = ?"); + + this.stmtL0GetMeta = this.db.prepare(` + SELECT session_key, session_id, role, message_text, recorded_at, timestamp + FROM l0_conversations WHERE record_id = ? + `); + + if (this.dimensions > 0) { + this.stmtL0SearchVec = this.db.prepare(` + SELECT record_id, distance + FROM l0_vec + WHERE embedding MATCH ? + AND k = ? + ORDER BY distance + `); + } + + // L0 query statements for L1 runner (newest-first + LIMIT to bound memory) + this.stmtL0QueryAll = this.db.prepare(` + SELECT record_id, session_key, session_id, role, message_text, recorded_at, timestamp + FROM l0_conversations + WHERE session_key = ? + ORDER BY timestamp DESC + LIMIT ? + `); + + this.stmtL0QueryAfter = this.db.prepare(` + SELECT record_id, session_key, session_id, role, message_text, recorded_at, timestamp + FROM l0_conversations + WHERE session_key = ? AND timestamp > ? + ORDER BY timestamp DESC + LIMIT ? + `); + + // ── FTS5 tables (best-effort — gracefully degrade if fts5 is not compiled in) ── + // Schema v2: `content` column stores jieba-segmented text (for indexing), + // `content_original` (UNINDEXED) stores the raw text (for display). + // If old v1 tables exist (no content_original column), drop + recreate. + try { + // ── Migrate old FTS5 tables (v1 → v2) ── + // v1 tables stored raw text in the `content` column. v2 stores segmented + // text in `content` and raw text in `content_original` / `message_text_original`. + // FTS5 virtual tables don't support ALTER TABLE ADD COLUMN, so we must + // drop and recreate. The data will be repopulated by `rebuildFtsIndex()`. + const needsFtsRebuild = this.migrateFtsTablesIfNeeded(); + + // L1 FTS5 virtual table (v2 schema) + this.db.exec(` + CREATE VIRTUAL TABLE IF NOT EXISTS l1_fts USING fts5( + content, + content_original UNINDEXED, + record_id UNINDEXED, + type UNINDEXED, + priority UNINDEXED, + scene_name UNINDEXED, + session_key UNINDEXED, + session_id UNINDEXED, + timestamp_str UNINDEXED, + timestamp_start UNINDEXED, + timestamp_end UNINDEXED, + metadata_json UNINDEXED + ) + `); + + // L0 FTS5 virtual table (v2 schema) + this.db.exec(` + CREATE VIRTUAL TABLE IF NOT EXISTS l0_fts USING fts5( + message_text, + message_text_original UNINDEXED, + record_id UNINDEXED, + session_key UNINDEXED, + session_id UNINDEXED, + role UNINDEXED, + recorded_at UNINDEXED, + timestamp UNINDEXED + ) + `); + + // L1 FTS prepared statements + this.stmtL1FtsInsert = this.db.prepare(` + INSERT INTO l1_fts (content, content_original, record_id, type, priority, scene_name, + session_key, session_id, timestamp_str, timestamp_start, timestamp_end, metadata_json) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `); + + this.stmtL1FtsDelete = this.db.prepare("DELETE FROM l1_fts WHERE record_id = ?"); + + this.stmtL1FtsSearch = this.db.prepare(` + SELECT record_id, content_original AS content, type, priority, scene_name, + session_key, session_id, timestamp_str, timestamp_start, timestamp_end, + metadata_json, + bm25(l1_fts) AS rank + FROM l1_fts + WHERE l1_fts MATCH ? + ORDER BY rank ASC + LIMIT ? + `); + + // L0 FTS prepared statements + this.stmtL0FtsInsert = this.db.prepare(` + INSERT INTO l0_fts (message_text, message_text_original, record_id, session_key, session_id, role, recorded_at, timestamp) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + `); + + this.stmtL0FtsDelete = this.db.prepare("DELETE FROM l0_fts WHERE record_id = ?"); + + this.stmtL0FtsSearch = this.db.prepare(` + SELECT record_id, message_text_original AS message_text, session_key, session_id, role, recorded_at, timestamp, + bm25(l0_fts) AS rank + FROM l0_fts + WHERE l0_fts MATCH ? + ORDER BY rank ASC + LIMIT ? + `); + + this.ftsAvailable = true; + this.logger?.info(`${TAG} FTS5 tables initialized (l1_fts, l0_fts) [schema v2 — jieba segmented]`); + + // Rebuild FTS index if migrated from v1 or tables were freshly created + if (needsFtsRebuild) { + this.rebuildFtsIndex(); + } + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + this.ftsAvailable = false; + this.logger?.warn( + `${TAG} FTS5 tables NOT available (fts5 may not be compiled in): ${message}. ` + + `FTS-based keyword search will be unavailable; recall will use in-memory scoring if needed.`, + ); + } + + // Save current embedding meta (write after schema is ready) + if (providerInfo) { + this.writeEmbeddingMeta({ + provider: providerInfo.provider, + model: providerInfo.model, + dimensions: this.dimensions, + }); + } + + // Mark vec0 tables as ready only when they were actually created + this.vecTablesReady = this.dimensions > 0; + // L1 query statements (for l1-reader) + const l1QueryCols = `record_id, content, type, priority, scene_name, session_key, session_id, + timestamp_str, timestamp_start, timestamp_end, + created_time, updated_time, metadata_json`; + + this.stmtQueryBySessionId = this.db.prepare(` + SELECT ${l1QueryCols} FROM l1_records + WHERE session_id = ? + ORDER BY updated_time ASC + `); + + this.stmtQueryBySessionIdSince = this.db.prepare(` + SELECT ${l1QueryCols} FROM l1_records + WHERE session_id = ? AND updated_time > ? + ORDER BY updated_time ASC + `); + + this.stmtQueryBySessionKey = this.db.prepare(` + SELECT ${l1QueryCols} FROM l1_records + WHERE session_key = ? + ORDER BY updated_time ASC + `); + + this.stmtQueryBySessionKeySince = this.db.prepare(` + SELECT ${l1QueryCols} FROM l1_records + WHERE session_key = ? AND updated_time > ? + ORDER BY updated_time ASC + `); + + this.stmtQueryAll = this.db.prepare(` + SELECT ${l1QueryCols} FROM l1_records + ORDER BY updated_time ASC + `); + + this.stmtQueryAllSince = this.db.prepare(` + SELECT ${l1QueryCols} FROM l1_records + WHERE updated_time > ? + ORDER BY updated_time ASC + `); + + this.logger?.info(`${TAG} Initialized (dimensions=${this.dimensions})`); + + return { needsReindex, reason: reindexReason }; + } + + // ── Embedding meta helpers ────────────────────────────── + + private readEmbeddingMeta(): EmbeddingMeta | null { + try { + const row = this.db + .prepare("SELECT value FROM embedding_meta WHERE key = ?") + .get("embedding_provider_info") as { value: string } | undefined; + if (!row) return null; + return JSON.parse(row.value) as EmbeddingMeta; + } catch { + return null; + } + } + + private writeEmbeddingMeta(meta: EmbeddingMeta): void { + this.db.prepare( + "INSERT INTO embedding_meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value=excluded.value", + ).run("embedding_provider_info", JSON.stringify(meta)); + } + + /** Allowed table names for row counting (whitelist to prevent SQL injection). */ + private static readonly COUNTABLE_TABLES = new Set(["l1_records", "l0_conversations"]); + + /** + * Extra rows to retrieve from vec0 KNN search to compensate for legacy + * zero-vector placeholders that may still linger from older data. + */ + private static readonly ZERO_VEC_BUFFER = 10; + + /** Default result limit for FTS5 keyword searches. */ + private static readonly FTS_DEFAULT_LIMIT = 20; + + private tableRowCount(table: string): number { + if (!VectorStore.COUNTABLE_TABLES.has(table)) { + this.logger?.warn(`${TAG} tableRowCount: rejected unknown table name "${table}"`); + return 0; + } + try { + const row = this.db + .prepare(`SELECT COUNT(*) AS cnt FROM ${table}`) + .get() as { cnt: number } | undefined; + return row?.cnt ?? 0; + } catch { + return 0; + } + } + + /** + * Detect the embedding dimension of an existing vec0 table by inspecting + * the DDL stored in sqlite_master. Returns `null` if the table doesn't + * exist or the dimension cannot be determined. + * + * The vec0 DDL looks like: + * CREATE VIRTUAL TABLE l1_vec USING vec0(... embedding float[768] ...) + * We parse the number inside `float[N]`. + */ + private getVecTableDimensions(): number | null { + try { + const row = this.db + .prepare("SELECT sql FROM sqlite_master WHERE type='table' AND name=?") + .get("l1_vec") as { sql: string } | undefined; + if (!row?.sql) return null; + const match = row.sql.match(/float\[(\d+)\]/); + return match ? Number(match[1]) : null; + } catch { + return null; + } + } + + /** + * Drop both L1 and L0 vector virtual tables. + * Metadata tables (l1_records, l0_conversations) are preserved — only + * the vec0 tables need to be rebuilt with the new dimensions. + */ + private dropVectorTables(): void { + this.db.exec("DROP TABLE IF EXISTS l1_vec"); + this.db.exec("DROP TABLE IF EXISTS l0_vec"); + this.logger?.info(`${TAG} Dropped vector tables (l1_vec, l0_vec)`); + } + + /** + * Write or update a memory record (metadata + vector). + * Uses a manual transaction for atomicity. + * + * If `embedding` is `undefined` or a zero vector (all elements are 0), only + * the metadata row is written — the vec0 table is left untouched. This + * allows callers without an EmbeddingService to still persist metadata + FTS + * without constructing a throwaway zero-vector, and prevents placeholder + * zero vectors (from embedding-service failures) from polluting KNN search + * results with null / NaN distances. + * + * **Fault-tolerant**: catches all errors internally so that a vector store + * failure never propagates to the caller / main OpenClaw flow. + * Returns `true` on success, `false` on failure (logged as warning). + */ + upsert(record: MemoryRecord, embedding: Float32Array | undefined): boolean { + if (this.degraded) { + this.logger?.warn(`${TAG} [L1-upsert] SKIPPED (degraded mode) id=${record.id}`); + return false; + } + try { + const { id: recordId, timestamps } = record; + const tsStr = timestamps[0] ?? ""; + const tsStart = + timestamps.length > 0 + ? timestamps.reduce((a, b) => (a < b ? a : b)) + : tsStr; + const tsEnd = + timestamps.length > 0 + ? timestamps.reduce((a, b) => (a > b ? a : b)) + : tsStr; + + const skipVec = !embedding || embedding.every(v => v === 0) || !this.vecTablesReady; + + this.logger?.debug?.( + `${TAG} [L1-upsert] START id=${recordId}, type=${record.type}, ` + + `content="${record.content.slice(0, 60)}..."` + + (embedding + ? `, embeddingDims=${embedding.length}, ` + + `embeddingNorm=${Math.sqrt(Array.from(embedding).reduce((s, v) => s + v * v, 0)).toFixed(4)}` + + `${skipVec ? " (ZERO VECTOR or vec tables not ready — vec write will be skipped)" : ""}` + : " (no embedding — metadata-only write)"), + ); + + this.db.exec("BEGIN"); + try { + // Upsert metadata (INSERT OR UPDATE) + this.stmtUpsertMeta.run( + recordId, + record.content, + record.type, + record.priority, + record.scene_name, + record.sessionKey, + record.sessionId, + tsStr, + tsStart, + tsEnd, + record.createdAt, + record.updatedAt, + JSON.stringify(record.metadata), + ); + + if (!skipVec) { + // vec0 does not support ON CONFLICT → delete then insert + this.stmtDeleteVec!.run(recordId); + this.stmtInsertVec!.run(recordId, Buffer.from(embedding!.buffer), record.updatedAt); + } else { + this.logger?.debug?.( + `${TAG} [L1-upsert] Skipping vec write (${embedding ? "zero vector" : "no embedding"}) id=${recordId}`, + ); + } + + // Sync FTS5 (delete + re-insert to handle updates) + if (this.ftsAvailable) { + try { + this.stmtL1FtsDelete.run(recordId); + this.stmtL1FtsInsert.run( + tokenizeForFts(record.content), // content — segmented for indexing + record.content, // content_original — raw for display + recordId, + record.type, + record.priority, + record.scene_name, + record.sessionKey, + record.sessionId, + tsStr, + tsStart, + tsEnd, + JSON.stringify(record.metadata), + ); + } catch (ftsErr) { + // FTS write failure is non-fatal — log and continue + this.logger?.warn( + `${TAG} [L1-upsert] FTS write failed (non-fatal) id=${recordId}: ${ftsErr instanceof Error ? ftsErr.message : String(ftsErr)}`, + ); + } + } + + this.db.exec("COMMIT"); + } catch (err) { + try { + this.db.exec("ROLLBACK"); + } catch { /* ignore rollback errors */ } + throw err; + } + this.logger?.debug?.(`${TAG} [L1-upsert] OK id=${recordId}${skipVec ? " (meta-only)" : ""}`); + return true; + } catch (err) { + this.logger?.warn( + `${TAG} [L1-upsert] FAILED (non-fatal) id=${record.id}: ${err instanceof Error ? err.message : String(err)}`, + ); + return false; + } + } + + /** + * Vector similarity search (cosine distance). + * Returns top-k results sorted by similarity (highest first). + * + * **Fault-tolerant**: returns an empty array on any error (e.g. dimension + * mismatch, corrupted DB) so callers can fall back to keyword search. + */ + search(queryEmbedding: Float32Array, topK = 5): VectorSearchResult[] { + if (this.degraded || !this.vecTablesReady) { + if (this.degraded) this.logger?.warn(`${TAG} [L1-search] SKIPPED (degraded mode)`); + return []; + } + try { + // Over-retrieve to compensate for legacy zero-vector placeholders that + // may still exist in the vec0 table. New zero vectors are no longer + // inserted (upsert() skips vec write for zero vectors since v3.x), but + // older data may still contain them — they surface as NULL/NaN distance + // in KNN results. A small buffer of 10 is sufficient for remnants. + // NOTE: "AND distance IS NOT NULL" is NOT usable because vec0 does not + // support that constraint — it causes an empty result set. + const ZERO_VEC_BUFFER = 10; + const retrieveCount = topK + ZERO_VEC_BUFFER; + + this.logger?.debug?.( + `${TAG} [L1-search] START topK=${topK}, retrieveCount=${retrieveCount}, ` + + `queryEmbeddingDims=${queryEmbedding.length}, ` + + `queryNorm=${Math.sqrt(Array.from(queryEmbedding).reduce((s, v) => s + v * v, 0)).toFixed(4)}`, + ); + + const rows = this.stmtSearchVec!.all( + Buffer.from(queryEmbedding.buffer), + retrieveCount, + ) as Array<{ record_id: string; distance: number }>; + + this.logger?.debug?.(`${TAG} [L1-search] vec0 returned ${rows.length} candidate(s)`); + + if (rows.length === 0) return []; + + const results: VectorSearchResult[] = []; + + for (const { record_id, distance } of rows) { + // sqlite-vec returns null distance for zero vectors (cosine undefined when ‖v‖=0). + // Skip these — they are placeholder vectors from embedding-service-unavailable fallback. + if (distance == null || Number.isNaN(distance)) { + this.logger?.warn( + `${TAG} [L1-search] record_id=${record_id} has null/NaN distance (likely zero vector) — skipping`, + ); + continue; + } + + const meta = this.stmtGetMeta.get(record_id) as + | { + content: string; + type: string; + priority: number; + scene_name: string; + session_key: string; + session_id: string; + timestamp_str: string; + timestamp_start: string; + timestamp_end: string; + metadata_json: string; + } + | undefined; + + if (!meta) { + this.logger?.warn(`${TAG} [L1-search] record_id=${record_id} has vector but NO metadata (orphan)`); + continue; + } + + const score = 1.0 - distance; + this.logger?.debug?.( + `${TAG} [L1-search] HIT id=${record_id}, distance=${distance.toFixed(4)}, score=${score.toFixed(4)}, ` + + `type=${meta.type}, content="${meta.content.slice(0, 60)}..."`, + ); + + results.push({ + record_id, + content: meta.content, + type: meta.type, + priority: meta.priority, + scene_name: meta.scene_name, + score, + timestamp_str: meta.timestamp_str, + timestamp_start: meta.timestamp_start, + timestamp_end: meta.timestamp_end, + session_key: meta.session_key, + session_id: meta.session_id, + metadata_json: meta.metadata_json, + }); + } + + // Trim back to the caller's requested topK (we over-fetched above). + const trimmed = results.slice(0, topK); + this.logger?.info( + `${TAG} [L1-search] DONE returning ${trimmed.length} result(s) (from ${results.length} valid, ${rows.length} raw)`, + ); + return trimmed; + } catch (err) { + this.logger?.warn( + `${TAG} [L1-search] FAILED (non-fatal, returning empty): ${err instanceof Error ? err.message : String(err)}`, + ); + return []; + } + } + + /** + * Delete a single record (metadata + vector). + * + * **Fault-tolerant**: logs a warning on failure, never throws. + */ + delete(recordId: string): boolean { + if (this.degraded) return false; + try { + this.db.exec("BEGIN"); + try { + this.stmtDeleteMeta.run(recordId); + if (this.vecTablesReady) this.stmtDeleteVec!.run(recordId); + if (this.ftsAvailable) { + try { this.stmtL1FtsDelete.run(recordId); } catch { /* non-fatal */ } + } + this.db.exec("COMMIT"); + } catch (err) { + try { + this.db.exec("ROLLBACK"); + } catch { /* ignore rollback errors */ } + throw err; + } + return true; + } catch (err) { + this.logger?.warn( + `${TAG} delete failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`, + ); + return false; + } + } + + /** + * Delete multiple records (metadata + vector). + * + * **Fault-tolerant**: logs a warning on failure, never throws. + */ + deleteBatch(recordIds: string[]): boolean { + if (this.degraded) return false; + if (recordIds.length === 0) return true; + + try { + this.db.exec("BEGIN"); + try { + for (const id of recordIds) { + this.stmtDeleteMeta.run(id); + if (this.vecTablesReady) this.stmtDeleteVec!.run(id); + if (this.ftsAvailable) { + try { this.stmtL1FtsDelete.run(id); } catch { /* non-fatal */ } + } + } + this.db.exec("COMMIT"); + } catch (err) { + try { + this.db.exec("ROLLBACK"); + } catch { /* ignore rollback errors */ } + throw err; + } + return true; + } catch (err) { + this.logger?.warn( + `${TAG} deleteBatch failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`, + ); + return false; + } + } + + /** + * Get the total number of L1 records in the store. + * + * **Fault-tolerant**: returns 0 on failure. + * TTL cleanup by updated_time. + * + * Deletes expired rows from l1_records and matching vectors from l1_vec + * in a single transaction to guarantee consistency. + */ + deleteL1ExpiredByUpdatedTime(cutoffIso: string): number { + if (this.degraded) { + this.logger?.warn(`${TAG} [deleteExpired] SKIPPED (degraded mode)`); + return 0; + } + try { + const row = this.db.prepare( + "SELECT COUNT(*) AS cnt FROM l1_records WHERE updated_time != '' AND updated_time < ?", + ).get(cutoffIso) as { cnt: number } | undefined; + const expiredCount = row?.cnt ?? 0; + if (expiredCount <= 0) return 0; + + this.db.exec("BEGIN"); + try { + if (this.vecTablesReady) { + this.db.prepare( + "DELETE FROM l1_vec WHERE updated_time != '' AND updated_time < ?", + ).run(cutoffIso); + } + this.db.prepare( + "DELETE FROM l1_records WHERE updated_time != '' AND updated_time < ?", + ).run(cutoffIso); + this.db.exec("COMMIT"); + return expiredCount; + } catch (err) { + try { + this.db.exec("ROLLBACK"); + } catch { /* ignore rollback errors */ } + throw err; + } + } catch (err) { + this.logger?.warn( + `${TAG} deleteL1ExpiredByUpdatedTime failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`, + ); + return 0; + } + } + + /** + * Get the total number of records in the store. + */ + count(): number { + if (this.degraded) return 0; + try { + const row = this.db + .prepare("SELECT COUNT(*) AS cnt FROM l1_records") + .get() as { cnt: number }; + this.logger?.debug?.(`${TAG} [L1-count] total=${row.cnt}`); + return row.cnt; + } catch (err) { + this.logger?.warn( + `${TAG} count failed (non-fatal, returning 0): ${err instanceof Error ? err.message : String(err)}`, + ); + return 0; + } + } + + /** + * Query L1 records with optional session and time filters. + * + * Uses the composite index `idx_l1_session_updated(session_id, updated_time)` + * for efficient filtering. All timestamps are compared as UTC ISO 8601 strings. + * + * **Fault-tolerant**: returns an empty array on any error (degraded mode, DB issues). + */ + queryL1Records(filter?: L1QueryFilter): L1RecordRow[] { + if (this.degraded) { + this.logger?.warn(`${TAG} [L1-query] SKIPPED (degraded mode)`); + return []; + } + try { + const { sessionKey, sessionId, updatedAfter } = filter ?? {}; + + let raw: Record[]; + + // Priority: sessionId > sessionKey (sessionId is more specific) + if (sessionId && updatedAfter) { + raw = this.stmtQueryBySessionIdSince.all(sessionId, updatedAfter) as Record[]; + } else if (sessionId) { + raw = this.stmtQueryBySessionId.all(sessionId) as Record[]; + } else if (sessionKey && updatedAfter) { + raw = this.stmtQueryBySessionKeySince.all(sessionKey, updatedAfter) as Record[]; + } else if (sessionKey) { + raw = this.stmtQueryBySessionKey.all(sessionKey) as Record[]; + } else if (updatedAfter) { + raw = this.stmtQueryAllSince.all(updatedAfter) as Record[]; + } else { + raw = this.stmtQueryAll.all() as Record[]; + } + + // Runtime sanity check: verify first row has expected columns (guards against schema drift) + if (raw.length > 0 && !("record_id" in raw[0] && "content" in raw[0])) { + this.logger?.warn( + `${TAG} [L1-query] Schema mismatch: first row missing expected columns. ` + + `Got keys: [${Object.keys(raw[0]).join(", ")}]`, + ); + return []; + } + + const rows = raw as unknown as L1RecordRow[]; + + this.logger?.info( + `${TAG} [L1-query] filter={sessionKey=${sessionKey ?? "(all)"}, sessionId=${sessionId ?? "(all)"}, updatedAfter=${updatedAfter ?? "(none)"}}, ` + + `returned ${rows.length} record(s)`, + ); + return rows; + } catch (err) { + this.logger?.warn( + `${TAG} [L1-query] FAILED (non-fatal, returning empty): ${err instanceof Error ? err.message : String(err)}` + ); + return []; + } + } + + // ── L0 operations ────────────────────────────────── + + /** + * Write or update an L0 single-message record (metadata + vector). + * Uses a manual transaction for atomicity. + * + * If `embedding` is `undefined` or a zero vector (all elements are 0), only + * the metadata row (`l0_conversations`) is written — the vec0 table + * (`l0_vec`) is left untouched. This allows callers without an + * EmbeddingService to still persist metadata + FTS without constructing a + * throwaway zero-vector, and prevents placeholder zero vectors (from + * embedding-service failures) from polluting KNN search results. + * + * **Fault-tolerant**: catches all errors internally, never throws. + * Returns `true` on success, `false` on failure (logged as warning). + */ + upsertL0(record: L0VectorRecord, embedding: Float32Array | undefined): boolean { + if (this.degraded) { + this.logger?.warn(`${TAG} [L0-upsert] SKIPPED (degraded mode) id=${record.id}`); + return false; + } + try { + const skipVec = !embedding || embedding.every(v => v === 0) || !this.vecTablesReady; + + this.logger?.debug?.( + `${TAG} [L0-upsert] START id=${record.id}, session=${record.sessionKey}, role=${record.role}, ` + + `text="${record.messageText.slice(0, 60)}..."` + + (embedding + ? `, embeddingDims=${embedding.length}, ` + + `embeddingNorm=${Math.sqrt(Array.from(embedding).reduce((s, v) => s + v * v, 0)).toFixed(4)}` + + `${skipVec ? " (ZERO VECTOR or vec tables not ready — vec write will be skipped)" : ""}` + : " (no embedding — metadata-only write)"), + ); + + this.db.exec("BEGIN"); + try { + this.stmtL0UpsertMeta.run( + record.id, + record.sessionKey, + record.sessionId, + record.role, + record.messageText, + record.recordedAt, + record.timestamp, + ); + + if (!skipVec) { + // vec0 does not support ON CONFLICT → delete then insert + this.stmtL0DeleteVec!.run(record.id); + this.stmtL0InsertVec!.run(record.id, Buffer.from(embedding!.buffer), record.recordedAt); + } else { + this.logger?.debug?.( + `${TAG} [L0-upsert] Skipping vec write (${embedding ? "zero vector" : "no embedding"}) id=${record.id}`, + ); + } + + // Sync FTS5 (delete + re-insert to handle updates) + if (this.ftsAvailable) { + try { + this.stmtL0FtsDelete.run(record.id); + this.stmtL0FtsInsert.run( + tokenizeForFts(record.messageText), // message_text — segmented for indexing + record.messageText, // message_text_original — raw for display + record.id, + record.sessionKey, + record.sessionId, + record.role, + record.recordedAt, + record.timestamp, + ); + } catch (ftsErr) { + // FTS write failure is non-fatal — log and continue + this.logger?.warn( + `${TAG} [L0-upsert] FTS write failed (non-fatal) id=${record.id}: ${ftsErr instanceof Error ? ftsErr.message : String(ftsErr)}`, + ); + } + } + + this.db.exec("COMMIT"); + } catch (err) { + try { + this.db.exec("ROLLBACK"); + } catch { /* ignore rollback errors */ } + throw err; + } + this.logger?.debug?.(`${TAG} [L0-upsert] OK id=${record.id}${skipVec ? " (meta-only)" : ""}`); + return true; + } catch (err) { + this.logger?.warn( + `${TAG} [L0-upsert] FAILED (non-fatal) id=${record.id}: ${err instanceof Error ? err.message : String(err)}`, + ); + return false; + } + } + + /** + * Update ONLY the vector embedding for an existing L0 record. + * The metadata row must already exist in l0_conversations (written by upsertL0). + * + * This is used by the background embedding task in auto-capture: + * 1. upsertL0() writes metadata + FTS synchronously (no embedding) + * 2. Background task calls embedBatch() then updateL0Embedding() for each record + * + * **Fault-tolerant**: catches all errors internally, never throws. + * Returns `true` on success, `false` on failure. + */ + updateL0Embedding(recordId: string, embedding: Float32Array): boolean { + if (this.degraded || !this.vecTablesReady) { + return false; + } + if (!embedding || embedding.every(v => v === 0)) { + this.logger?.debug?.(`${TAG} [L0-update-embedding] Skipping zero vector for ${recordId}`); + return false; + } + try { + // Look up recorded_at from metadata for the vec0 row + const meta = this.stmtL0GetMeta.get(recordId) as { recorded_at: string } | undefined; + if (!meta) { + this.logger?.warn(`${TAG} [L0-update-embedding] No metadata found for ${recordId}, skipping`); + return false; + } + + this.db.exec("BEGIN"); + try { + this.stmtL0DeleteVec!.run(recordId); + this.stmtL0InsertVec!.run(recordId, Buffer.from(embedding.buffer), meta.recorded_at); + this.db.exec("COMMIT"); + } catch (err) { + try { this.db.exec("ROLLBACK"); } catch { /* ignore */ } + throw err; + } + return true; + } catch (err) { + this.logger?.warn( + `${TAG} [L0-update-embedding] FAILED (non-fatal) id=${recordId}: ${err instanceof Error ? err.message : String(err)}`, + ); + return false; + } + } + + /** + * Vector similarity search on L0 individual messages (cosine distance). + * Returns top-k results sorted by similarity (highest first). + * + * **Fault-tolerant**: returns an empty array on any error. + */ + searchL0(queryEmbedding: Float32Array, topK = 5): L0VectorSearchResult[] { + if (this.degraded || !this.vecTablesReady) { + if (this.degraded) this.logger?.warn(`${TAG} [L0-search] SKIPPED (degraded mode)`); + return []; + } + try { + // Over-retrieve to compensate for legacy zero-vector placeholders that + // may still exist in the vec0 table. New zero vectors are no longer + // inserted (upsertL0() skips vec write for zero vectors since v3.x), but + // older data may still contain them — they surface as NULL/NaN distance + // in KNN results. + // NOTE: "AND distance IS NOT NULL" is NOT usable because vec0 does not + // support that constraint — it causes an empty result set. + const retrieveCount = topK + VectorStore.ZERO_VEC_BUFFER; + + this.logger?.debug?.( + `${TAG} [L0-search] START topK=${topK}, retrieveCount=${retrieveCount}, ` + + `queryEmbeddingDims=${queryEmbedding.length}, ` + + `queryNorm=${Math.sqrt(Array.from(queryEmbedding).reduce((s, v) => s + v * v, 0)).toFixed(4)}`, + ); + + const rows = this.stmtL0SearchVec!.all( + Buffer.from(queryEmbedding.buffer), + retrieveCount, + ) as Array<{ record_id: string; distance: number }>; + + this.logger?.debug?.(`${TAG} [L0-search] vec0 returned ${rows.length} candidate(s)`); + + if (rows.length === 0) return []; + + const results: L0VectorSearchResult[] = []; + + for (const { record_id, distance } of rows) { + // sqlite-vec returns null distance for zero vectors (cosine undefined when ‖v‖=0). + // Skip these — they are placeholder vectors from embedding-service-unavailable fallback. + if (distance == null || Number.isNaN(distance)) { + this.logger?.warn( + `${TAG} [L0-search] record_id=${record_id} has null/NaN distance (likely zero vector) — skipping`, + ); + continue; + } + + const meta = this.stmtL0GetMeta.get(record_id) as + | { + session_key: string; + session_id: string; + role: string; + message_text: string; + recorded_at: string; + timestamp: number; + } + | undefined; + + if (!meta) { + this.logger?.warn(`${TAG} [L0-search] record_id=${record_id} has vector but NO metadata (orphan)`); + continue; + } + + const score = 1.0 - distance; + this.logger?.debug?.( + `${TAG} [L0-search] HIT id=${record_id}, distance=${distance.toFixed(4)}, score=${score.toFixed(4)}, ` + + `role=${meta.role}, session=${meta.session_key}, text="${meta.message_text.slice(0, 60)}..."`, + ); + + results.push({ + record_id, + session_key: meta.session_key, + session_id: meta.session_id, + role: meta.role, + message_text: meta.message_text, + score, + recorded_at: meta.recorded_at, + timestamp: meta.timestamp ?? 0, + }); + } + + // Trim back to the caller's requested topK (we over-fetched above). + const trimmed = results.slice(0, topK); + this.logger?.info( + `${TAG} [L0-search] DONE returning ${trimmed.length} result(s) (from ${results.length} valid, ${rows.length} raw)`, + ); + return trimmed; + } catch (err) { + this.logger?.warn( + `${TAG} [L0-search] FAILED (non-fatal, returning empty): ${err instanceof Error ? err.message : String(err)}`, + ); + return []; + } + } + + /** + * Delete a single L0 record (metadata + vector). + * + * **Fault-tolerant**: logs a warning on failure, never throws. + */ + deleteL0(recordId: string): boolean { + if (this.degraded) return false; + try { + this.db.exec("BEGIN"); + try { + this.stmtL0DeleteMeta.run(recordId); + if (this.vecTablesReady) this.stmtL0DeleteVec!.run(recordId); + if (this.ftsAvailable) { + try { this.stmtL0FtsDelete.run(recordId); } catch { /* non-fatal */ } + } + this.db.exec("COMMIT"); + } catch (err) { + try { + this.db.exec("ROLLBACK"); + } catch { /* ignore rollback errors */ } + throw err; + } + return true; + } catch (err) { + this.logger?.warn( + `${TAG} deleteL0 failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`, + ); + return false; + } + } + + /** + * TTL cleanup by recorded_at (ISO string) for L0 records. + * + * Deletes expired rows from l0_conversations and matching vectors from l0_vec + * in a single transaction to guarantee consistency. + */ + deleteL0ExpiredByRecordedAt(cutoffIso: string): number { + if (this.degraded) { + this.logger?.warn(`${TAG} [deleteExpiredL0] SKIPPED (degraded mode)`); + return 0; + } + + try { + const row = this.db.prepare( + "SELECT COUNT(*) AS cnt FROM l0_conversations WHERE recorded_at != '' AND recorded_at < ?", + ).get(cutoffIso) as { cnt: number } | undefined; + const expiredCount = row?.cnt ?? 0; + if (expiredCount <= 0) return 0; + + this.db.exec("BEGIN"); + try { + if (this.vecTablesReady) { + this.db.prepare( + "DELETE FROM l0_vec WHERE recorded_at != '' AND recorded_at < ?", + ).run(cutoffIso); + } + this.db.prepare( + "DELETE FROM l0_conversations WHERE recorded_at != '' AND recorded_at < ?", + ).run(cutoffIso); + this.db.exec("COMMIT"); + return expiredCount; + } catch (err) { + try { + this.db.exec("ROLLBACK"); + } catch { /* ignore rollback errors */ } + throw err; + } + } catch (err) { + this.logger?.warn( + `${TAG} deleteL0ExpiredByRecordedAt failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`, + ); + return 0; + } + } + + /** + * Get the total number of L0 message records in the store. + * + * **Fault-tolerant**: returns 0 on failure. + */ + countL0(): number { + if (this.degraded) return 0; + try { + const row = this.db + .prepare("SELECT COUNT(*) AS cnt FROM l0_conversations") + .get() as { cnt: number }; + this.logger?.debug?.(`${TAG} [L0-count] total=${row.cnt}`); + return row.cnt; + } catch (err) { + this.logger?.warn( + `${TAG} countL0 failed (non-fatal, returning 0): ${err instanceof Error ? err.message : String(err)}`, + ); + return 0; + } + } + + // ── Re-index operations ────────────────────────────────── + + /** + * Get all L1 record texts for re-embedding. + * Returns record_id → content pairs. + */ + getAllL1Texts(): Array<{ record_id: string; content: string; updated_time: string }> { + if (this.degraded) return []; + try { + return this.db + .prepare("SELECT record_id, content, updated_time FROM l1_records") + .all() as Array<{ record_id: string; content: string; updated_time: string }>; + } catch (err) { + this.logger?.warn( + `${TAG} getAllL1Texts failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`, + ); + return []; + } + } + + /** + * Get all L0 message texts for re-embedding. + * Returns record_id → message_text/recorded_at tuples. + */ + getAllL0Texts(): Array<{ record_id: string; message_text: string; recorded_at: string }> { + if (this.degraded) return []; + try { + return this.db + .prepare("SELECT record_id, message_text, recorded_at FROM l0_conversations") + .all() as Array<{ record_id: string; message_text: string; recorded_at: string }>; + } catch (err) { + this.logger?.warn( + `${TAG} getAllL0Texts failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`, + ); + return []; + } + } + + /** + * Re-embed all existing L1 and L0 texts with a new embedding function. + * + * This is called after `init()` returns `needsReindex: true` — the vector + * tables have already been dropped and re-created with the correct dimensions. + * This method reads every text from the metadata tables and writes fresh + * embeddings into the new vector tables. + * + * @param embedFn A function that converts text → Float32Array embedding. + * @param onProgress Optional callback for progress reporting. + */ + async reindexAll( + embedFn: (text: string) => Promise, + onProgress?: (done: number, total: number, layer: "L1" | "L0") => void, + ): Promise<{ l1Count: number; l0Count: number }> { + if (this.degraded || !this.vecTablesReady) { + if (this.degraded) this.logger?.warn(`${TAG} reindexAll skipped: VectorStore is in degraded mode`); + return { l1Count: 0, l0Count: 0 }; + } + + try { + // ── Re-embed L1 ── + const l1Rows = this.getAllL1Texts(); + let l1Done = 0; + for (const { record_id, content, updated_time } of l1Rows) { + try { + const embedding = await embedFn(content); + // Wrap delete+insert in a transaction to prevent orphan vectors + this.db.exec("BEGIN"); + try { + this.stmtDeleteVec!.run(record_id); + this.stmtInsertVec!.run(record_id, Buffer.from(embedding.buffer), updated_time); + this.db.exec("COMMIT"); + } catch (txErr) { + try { this.db.exec("ROLLBACK"); } catch { /* ignore */ } + throw txErr; + } + } catch (err) { + this.logger?.warn?.( + `${TAG} reindex L1 skip ${record_id}: ${err instanceof Error ? err.message : String(err)}`, + ); + } + l1Done++; + onProgress?.(l1Done, l1Rows.length, "L1"); + } + + // ── Re-embed L0 ── + const l0Rows = this.getAllL0Texts(); + let l0Done = 0; + for (const { record_id, message_text, recorded_at } of l0Rows) { + try { + const embedding = await embedFn(message_text); + // Wrap delete+insert in a transaction to prevent orphan vectors + this.db.exec("BEGIN"); + try { + this.stmtL0DeleteVec!.run(record_id); + this.stmtL0InsertVec!.run(record_id, Buffer.from(embedding.buffer), recorded_at); + this.db.exec("COMMIT"); + } catch (txErr) { + try { this.db.exec("ROLLBACK"); } catch { /* ignore */ } + throw txErr; + } + } catch (err) { + this.logger?.warn?.( + `${TAG} reindex L0 skip ${record_id}: ${err instanceof Error ? err.message : String(err)}`, + ); + } + l0Done++; + onProgress?.(l0Done, l0Rows.length, "L0"); + } + + this.logger?.info( + `${TAG} Reindex complete: L1=${l1Done}/${l1Rows.length}, L0=${l0Done}/${l0Rows.length}`, + ); + + return { l1Count: l1Done, l0Count: l0Done }; + } catch (err) { + this.logger?.error( + `${TAG} reindexAll failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`, + ); + return { l1Count: 0, l0Count: 0 }; + } + } + + // ── L0 query operations (for L1 runner) ────────────────────────────────── + + /** + * Query L0 messages for a given session key, optionally filtered by timestamp cursor. + * Returns messages ordered by timestamp ASC (chronological). + * + * Used by L1 runner to read L0 data from DB instead of JSONL files. + */ + queryL0ForL1( + sessionKey: string, + afterTimestamp?: number, + limit = 50, + ): Array<{ + record_id: string; + session_key: string; + session_id: string; + role: string; + message_text: string; + recorded_at: string; + timestamp: number; + }> { + if (this.degraded) { + this.logger?.warn(`${TAG} [L0-query] SKIPPED (degraded mode)`); + return []; + } + try { + // Query newest-first (DESC) with LIMIT, then reverse to chronological order + let rows: Array>; + if (afterTimestamp && afterTimestamp > 0) { + rows = this.stmtL0QueryAfter.all(sessionKey, afterTimestamp, limit) as Array>; + } else { + rows = this.stmtL0QueryAll.all(sessionKey, limit) as Array>; + } + + this.logger?.info( + `${TAG} [L0-query] session=${sessionKey}, afterTs=${afterTimestamp ?? "(all)"}, ` + + `limit=${limit}, returned ${rows.length} row(s)`, + ); + + // Reverse: SQL returns newest-first (DESC), callers expect chronological order + return rows.map((r) => ({ + record_id: r.record_id as string, + session_key: r.session_key as string, + session_id: (r.session_id as string) || "", + role: r.role as string, + message_text: r.message_text as string, + recorded_at: (r.recorded_at as string) || "", + timestamp: (r.timestamp as number) || 0, + })).reverse(); + } catch (err) { + this.logger?.warn( + `${TAG} [L0-query] FAILED (non-fatal, returning empty): ${err instanceof Error ? err.message : String(err)}`, + ); + return []; + } + } + + /** + * Query L0 messages for a given session key, grouped by session_id. + * Each group's messages are in chronological order (timestamp ASC). + * Groups are sorted by earliest message timestamp. + * + * Used by L1 runner to replace readConversationMessagesGroupedBySessionId(). + */ + queryL0GroupedBySessionId( + sessionKey: string, + afterTimestamp?: number, + limit = 50, + ): Array<{ sessionId: string; messages: Array<{ id: string; role: string; content: string; timestamp: number }> }> { + if (this.degraded) { + this.logger?.warn(`${TAG} [L0-query-grouped] SKIPPED (degraded mode)`); + return []; + } + try { + const rows = this.queryL0ForL1(sessionKey, afterTimestamp, limit); + + // Group by session_id + const groupMap = new Map>(); + for (const row of rows) { + const sid = row.session_id || ""; + let group = groupMap.get(sid); + if (!group) { + group = []; + groupMap.set(sid, group); + } + group.push({ + id: row.record_id, + role: row.role, + content: row.message_text, + timestamp: row.timestamp, + }); + } + + // Convert to array, sorted by earliest message timestamp + const groups: Array<{ sessionId: string; messages: Array<{ id: string; role: string; content: string; timestamp: number }> }> = []; + for (const [sessionId, messages] of groupMap) { + if (messages.length > 0) { + groups.push({ sessionId, messages }); + } + } + groups.sort((a, b) => a.messages[0].timestamp - b.messages[0].timestamp); + + this.logger?.info( + `${TAG} [L0-query-grouped] session=${sessionKey}, afterTs=${afterTimestamp ?? "(all)"}, ` + + `${rows.length} messages across ${groups.length} group(s)`, + ); + + return groups; + } catch (err) { + this.logger?.warn( + `${TAG} [L0-query-grouped] FAILED (non-fatal, returning empty): ${err instanceof Error ? err.message : String(err)}`, + ); + return []; + } + } + + // ── FTS5 search operations ────────────────────────────────── + + /** + * Whether FTS5 full-text search is available. + * When `false`, callers should skip keyword-based recall entirely. + */ + isFtsAvailable(): boolean { + return this.ftsAvailable; + } + + /** + * FTS5 keyword search on L1 records. + * Returns top-`limit` results sorted by BM25 relevance (highest first). + * + * @param ftsQuery A pre-built FTS5 MATCH expression (from `buildFtsQuery()`). + * @param limit Maximum number of results to return. + * + * **Fault-tolerant**: returns an empty array on any error. + */ + ftsSearchL1(ftsQuery: string, limit = 20): FtsSearchResult[] { + if (this.degraded || !this.ftsAvailable) return []; + try { + const rows = this.stmtL1FtsSearch.all(ftsQuery, limit) as Array<{ + record_id: string; + content: string; + type: string; + priority: number; + scene_name: string; + session_key: string; + session_id: string; + timestamp_str: string; + timestamp_start: string; + timestamp_end: string; + metadata_json: string; + rank: number; + }>; + + return rows.map((r) => ({ + record_id: r.record_id, + content: r.content, + type: r.type, + priority: r.priority, + scene_name: r.scene_name, + score: bm25RankToScore(r.rank), + timestamp_str: r.timestamp_str, + timestamp_start: r.timestamp_start, + timestamp_end: r.timestamp_end, + session_key: r.session_key, + session_id: r.session_id, + metadata_json: r.metadata_json, + })); + } catch (err) { + this.logger?.warn( + `${TAG} [L1-fts-search] FAILED (non-fatal, returning empty): ${err instanceof Error ? err.message : String(err)}`, + ); + return []; + } + } + + /** + * FTS5 keyword search on L0 conversation messages. + * Returns top-`limit` results sorted by BM25 relevance (highest first). + * + * @param ftsQuery A pre-built FTS5 MATCH expression (from `buildFtsQuery()`). + * @param limit Maximum number of results to return. + * + * **Fault-tolerant**: returns an empty array on any error. + */ + ftsSearchL0(ftsQuery: string, limit = VectorStore.FTS_DEFAULT_LIMIT): L0FtsSearchResult[] { + if (this.degraded || !this.ftsAvailable) return []; + try { + const rows = this.stmtL0FtsSearch.all(ftsQuery, limit) as Array<{ + record_id: string; + message_text: string; + session_key: string; + session_id: string; + role: string; + recorded_at: string; + timestamp: number; + rank: number; + }>; + + return rows.map((r) => ({ + record_id: r.record_id, + session_key: r.session_key, + session_id: r.session_id, + role: r.role, + message_text: r.message_text, + score: bm25RankToScore(r.rank), + recorded_at: r.recorded_at, + timestamp: r.timestamp ?? 0, + })); + } catch (err) { + this.logger?.warn( + `${TAG} [L0-fts-search] FAILED (non-fatal, returning empty): ${err instanceof Error ? err.message : String(err)}`, + ); + return []; + } + } + + // ── FTS5 migration & rebuild ────────────────────────────────────────────── + + /** + * Detect old FTS5 v1 schema (no `content_original` column) and drop the + * tables so they can be recreated with the v2 schema. + * + * FTS5 virtual tables do NOT support `ALTER TABLE ADD COLUMN`, so the only + * migration path is DROP + recreate + repopulate. + * + * @returns `true` if migration was performed (= FTS index needs rebuilding). + * @internal + */ + private migrateFtsTablesIfNeeded(): boolean { + try { + // Check if l1_fts exists at all + const l1Exists = this.db + .prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='l1_fts'") + .get(); + if (!l1Exists) { + // Fresh install — tables will be created with v2 schema. + // Still need rebuild if there's existing data in l1_records. + const hasData = this.db.prepare("SELECT 1 FROM l1_records LIMIT 1").get(); + return !!hasData; + } + + // Check if the v2 column `content_original` exists. + // FTS5 tables appear in pragma_table_info with their column names. + const cols = this.db + .prepare("SELECT name FROM pragma_table_info('l1_fts')") + .all() as Array<{ name: string }>; + const hasV2Col = cols.some((c) => c.name === "content_original"); + + if (hasV2Col) { + return false; // Already v2 — no migration needed + } + + // v1 → v2: drop both FTS tables (data will be repopulated by rebuildFtsIndex) + this.logger?.info(`${TAG} Migrating FTS5 tables from v1 to v2 (jieba segmented)`); + this.db.exec("DROP TABLE IF EXISTS l1_fts"); + this.db.exec("DROP TABLE IF EXISTS l0_fts"); + return true; + } catch (err) { + this.logger?.warn( + `${TAG} FTS migration check failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`, + ); + return false; + } + } + + /** + * Rebuild the FTS5 index from scratch by reading all records from the + * metadata tables and re-inserting them with jieba-segmented text. + * + * Called automatically after: + * - Schema migration from v1 to v2 + * - Fresh table creation when existing data exists + * + * Safe to call multiple times (idempotent — clears FTS tables first). + */ + rebuildFtsIndex(): void { + if (!this.ftsAvailable) return; + + try { + this.logger?.info(`${TAG} Rebuilding FTS5 index with jieba segmentation…`); + + // ── Rebuild L1 FTS ── + // Clear existing FTS data + this.db.exec("DELETE FROM l1_fts"); + + // Read all L1 records from metadata table + const l1Rows = this.db + .prepare(` + SELECT record_id, content, type, priority, scene_name, + session_key, session_id, timestamp_str, timestamp_start, timestamp_end, metadata_json + FROM l1_records + `) + .all() as Array<{ + record_id: string; + content: string; + type: string; + priority: number; + scene_name: string; + session_key: string; + session_id: string; + timestamp_str: string; + timestamp_start: string; + timestamp_end: string; + metadata_json: string; + }>; + + let l1Count = 0; + for (const r of l1Rows) { + try { + this.stmtL1FtsInsert.run( + tokenizeForFts(r.content), // content — segmented + r.content, // content_original — raw + r.record_id, + r.type, + r.priority, + r.scene_name, + r.session_key, + r.session_id, + r.timestamp_str, + r.timestamp_start, + r.timestamp_end, + r.metadata_json, + ); + l1Count++; + } catch (err) { + this.logger?.warn?.( + `${TAG} FTS rebuild skip L1 ${r.record_id}: ${err instanceof Error ? err.message : String(err)}`, + ); + } + } + + // ── Rebuild L0 FTS ── + this.db.exec("DELETE FROM l0_fts"); + + const l0Rows = this.db + .prepare(` + SELECT record_id, message_text, session_key, session_id, role, recorded_at, timestamp + FROM l0_conversations + `) + .all() as Array<{ + record_id: string; + message_text: string; + session_key: string; + session_id: string; + role: string; + recorded_at: string; + timestamp: number; + }>; + + let l0Count = 0; + for (const r of l0Rows) { + try { + this.stmtL0FtsInsert.run( + tokenizeForFts(r.message_text), // message_text — segmented + r.message_text, // message_text_original — raw + r.record_id, + r.session_key, + r.session_id, + r.role, + r.recorded_at, + r.timestamp, + ); + l0Count++; + } catch (err) { + this.logger?.warn?.( + `${TAG} FTS rebuild skip L0 ${r.record_id}: ${err instanceof Error ? err.message : String(err)}`, + ); + } + } + + this.logger?.info( + `${TAG} FTS5 rebuild complete: L1=${l1Count}/${l1Rows.length}, L0=${l0Count}/${l0Rows.length}`, + ); + } catch (err) { + this.logger?.warn( + `${TAG} FTS5 rebuild failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`, + ); + } + } + + /** + * Close the database connection. + * Should be called on shutdown. Idempotent — safe to call multiple times. + */ + close(): void { + if (this.closed) return; + this.closed = true; + try { + this.db.close(); + } catch (err) { + this.logger?.warn?.( + `${TAG} Error closing database: ${err instanceof Error ? err.message : String(err)}`, + ); + } + } +} diff --git a/src/tools/conversation-search.ts b/src/tools/conversation-search.ts new file mode 100644 index 0000000..8947e0e --- /dev/null +++ b/src/tools/conversation-search.ts @@ -0,0 +1,279 @@ +/** + * conversation_search tool: Agent-callable tool for searching L0 conversation records. + * + * Supports three search strategies with automatic degradation: + * 1. **hybrid** (default) — FTS5 keyword + vector embedding in parallel, + * merged via Reciprocal Rank Fusion (RRF). + * 2. **embedding** — pure vector similarity (when FTS5 is unavailable). + * 3. **fts** — pure FTS5 keyword search (when embedding is unavailable). + * + * The tool is registered via `api.registerTool()` in index.ts. + */ + +import type { VectorStore, L0VectorSearchResult } from "../store/vector-store.js"; +import { buildFtsQuery } from "../store/vector-store.js"; +import type { EmbeddingService } from "../store/embedding.js"; + +// ============================ +// Types +// ============================ + +interface Logger { + debug?: (message: string) => void; + info: (message: string) => void; + warn: (message: string) => void; + error: (message: string) => void; +} + +export interface ConversationSearchResultItem { + id: string; + session_key: string; + /** Role of the message sender: "user" or "assistant" */ + role: string; + /** Text content of this single message */ + content: string; + score: number; + recorded_at: string; +} + +export interface ConversationSearchResult { + results: ConversationSearchResultItem[]; + total: number; + /** Actual search strategy used: "hybrid", "embedding", "fts", or "none". */ + strategy: string; + /** Optional message, e.g. when embedding is not configured. */ + message?: string; +} + +const TAG = "[memory-tdai][tdai_conversation_search]"; + +// ============================ +// RRF (Reciprocal Rank Fusion) +// ============================ + +/** Standard RRF constant from the original RRF paper. */ +const RRF_K = 60; + +/** + * Merge multiple ranked lists of `ConversationSearchResultItem` via Reciprocal + * Rank Fusion. Items appearing in multiple lists get their RRF scores summed. + * + * Returns items sorted by descending RRF score. The `score` field of each + * returned item is replaced by the RRF score for consistent ranking semantics. + */ +function rrfMergeL0(...lists: ConversationSearchResultItem[][]): ConversationSearchResultItem[] { + const map = new Map(); + + for (const list of lists) { + for (let rank = 0; rank < list.length; rank++) { + const item = list[rank]; + const score = 1 / (RRF_K + rank + 1); + const existing = map.get(item.id); + if (existing) { + existing.rrfScore += score; + } else { + map.set(item.id, { item, rrfScore: score }); + } + } + } + + return [...map.values()] + .sort((a, b) => b.rrfScore - a.rrfScore) + .map(({ item, rrfScore }) => ({ ...item, score: rrfScore })); +} + +// ============================ +// Search implementation +// ============================ + +export async function executeConversationSearch(params: { + query: string; + limit: number; + sessionKey?: string; + vectorStore?: VectorStore; + embeddingService?: EmbeddingService; + logger?: Logger; +}): Promise { + const { + query, + limit, + sessionKey: sessionFilter, + vectorStore, + embeddingService, + logger, + } = params; + + logger?.debug?.( + `${TAG} CALLED: query="${query.slice(0, 100)}", limit=${limit}, ` + + `sessionFilter=${sessionFilter ?? "(none)"}, ` + + `vectorStore=${vectorStore ? "available" : "UNAVAILABLE"}, ` + + `embeddingService=${embeddingService ? "available" : "UNAVAILABLE"}`, + ); + + if (!query || query.trim().length === 0) { + logger?.debug?.(`${TAG} Empty query, returning empty`); + return { results: [], total: 0, strategy: "none" }; + } + + if (!vectorStore) { + logger?.warn?.(`${TAG} VectorStore not available`); + return { results: [], total: 0, strategy: "none" }; + } + + // ── Determine available capabilities ── + const hasEmbedding = !!embeddingService; + const hasFts = vectorStore.isFtsAvailable(); + + if (!hasEmbedding && !hasFts) { + logger?.warn?.(`${TAG} Neither EmbeddingService nor FTS5 available — cannot search`); + return { + results: [], + total: 0, + strategy: "none", + message: + "Embedding service is not configured and FTS is not available. " + + "Conversation search requires an embedding provider or FTS5 support. " + + "Please configure an embedding provider in the embedding.provider setting (e.g. openai_compatible).", + }; + } + + // ── Over-retrieve for later filtering and RRF merging ── + const candidateK = sessionFilter ? limit * 4 : limit * 3; + + // ── Run available search strategies in parallel ── + const [ftsItems, vecItems] = await Promise.all([ + // FTS5 keyword search on L0 + (async (): Promise => { + if (!hasFts) return []; + try { + const ftsQuery = buildFtsQuery(query); + if (!ftsQuery) { + logger?.debug?.(`${TAG} [hybrid-fts] No usable FTS tokens from query`); + return []; + } + logger?.debug?.(`${TAG} [hybrid-fts] FTS5 query: "${ftsQuery}"`); + const ftsResults = vectorStore.ftsSearchL0(ftsQuery, candidateK); + logger?.debug?.(`${TAG} [hybrid-fts] FTS5 returned ${ftsResults.length} candidates`); + return ftsResults.map((r) => ({ + id: r.record_id, + session_key: r.session_key, + role: r.role, + content: r.message_text, + score: r.score, + recorded_at: r.recorded_at, + })); + } catch (err) { + logger?.warn?.( + `${TAG} [hybrid-fts] FTS5 search failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`, + ); + return []; + } + })(), + + // Vector embedding search on L0 + (async (): Promise => { + if (!hasEmbedding) return []; + try { + logger?.debug?.(`${TAG} [hybrid-vec] Generating query embedding...`); + const queryEmbedding = await embeddingService!.embed(query); + logger?.debug?.( + `${TAG} [hybrid-vec] Embedding OK, dims=${queryEmbedding.length}, searching top-${candidateK}...`, + ); + const vecResults: L0VectorSearchResult[] = vectorStore.searchL0(queryEmbedding, candidateK); + logger?.debug?.(`${TAG} [hybrid-vec] Vector search returned ${vecResults.length} candidates`); + return vecResults.map((r) => ({ + id: r.record_id, + session_key: r.session_key, + role: r.role, + content: r.message_text, + score: r.score, + recorded_at: r.recorded_at, + })); + } catch (err) { + logger?.warn?.( + `${TAG} [hybrid-vec] Embedding search failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`, + ); + return []; + } + })(), + ]); + + // ── Determine effective strategy ── + const ftsOk = ftsItems.length > 0; + const vecOk = vecItems.length > 0; + let strategy: string; + + if (ftsOk && vecOk) { + strategy = "hybrid"; + } else if (vecOk) { + strategy = "embedding"; + } else if (ftsOk) { + strategy = "fts"; + } else { + logger?.debug?.(`${TAG} Both search paths returned 0 results`); + return { results: [], total: 0, strategy: hasEmbedding ? "embedding" : "fts" }; + } + + // ── Merge results ── + let results: ConversationSearchResultItem[]; + if (strategy === "hybrid") { + results = rrfMergeL0(ftsItems, vecItems); + logger?.debug?.( + `${TAG} [hybrid] RRF merged: fts=${ftsItems.length}, vec=${vecItems.length} → ${results.length} unique`, + ); + } else { + // Single-source: use whichever list has results (already sorted by score) + results = ftsOk ? ftsItems : vecItems; + } + + // ── Apply session key filter ── + if (sessionFilter) { + const preFilterCount = results.length; + results = results.filter((r) => r.session_key === sessionFilter); + logger?.debug?.(`${TAG} After session filter "${sessionFilter}": ${results.length}/${preFilterCount}`); + } + + // ── Trim to requested limit ── + const trimmed = results.slice(0, limit); + + logger?.debug?.( + `${TAG} RESULT (strategy=${strategy}): returning ${trimmed.length} messages ` + + `(scores: [${trimmed.map((r) => r.score.toFixed(3)).join(", ")}])`, + ); + + return { + results: trimmed, + total: trimmed.length, + strategy, + }; +} + +// ============================ +// Tool response formatter +// ============================ + +export function formatConversationSearchResponse(result: ConversationSearchResult): string { + if (result.message) { + return result.message; + } + if (result.results.length === 0) { + return "No matching conversation messages found."; + } + + const lines: string[] = [ + `Found ${result.total} matching message(s):`, + "", + ]; + + for (const item of result.results) { + const scoreStr = typeof item.score === "number" ? ` (score: ${item.score.toFixed(3)})` : ""; + const dateStr = item.recorded_at ? ` [${item.recorded_at}]` : ""; + lines.push(`---`); + lines.push(`**[${item.role}]** Session: ${item.session_key}${dateStr}${scoreStr}`); + lines.push(""); + lines.push(item.content); + lines.push(""); + } + + return lines.join("\n"); +} diff --git a/src/tools/memory-search.ts b/src/tools/memory-search.ts new file mode 100644 index 0000000..3725ced --- /dev/null +++ b/src/tools/memory-search.ts @@ -0,0 +1,290 @@ +/** + * memory_search tool: Agent-callable tool for searching L1 memory records. + * + * Supports three search strategies with automatic degradation: + * 1. **hybrid** (default) — FTS5 keyword + vector embedding in parallel, + * merged via Reciprocal Rank Fusion (RRF). + * 2. **embedding** — pure vector similarity (when FTS5 is unavailable). + * 3. **fts** — pure FTS5 keyword search (when embedding is unavailable). + * + * The tool is registered via `api.registerTool()` in index.ts. + */ + +import type { VectorStore, VectorSearchResult } from "../store/vector-store.js"; +import { buildFtsQuery } from "../store/vector-store.js"; +import type { EmbeddingService } from "../store/embedding.js"; + +// ============================ +// Types +// ============================ + +interface Logger { + debug?: (message: string) => void; + info: (message: string) => void; + warn: (message: string) => void; + error: (message: string) => void; +} + +export interface MemorySearchResultItem { + id: string; + content: string; + type: string; + priority: number; + scene_name: string; + score: number; + created_at: string; + updated_at: string; +} + +export interface MemorySearchResult { + results: MemorySearchResultItem[]; + total: number; + strategy: string; + /** Optional message, e.g. when embedding is not configured. */ + message?: string; +} + +const TAG = "[memory-tdai][tdai_memory_search]"; + +// ============================ +// RRF (Reciprocal Rank Fusion) +// ============================ + +/** Standard RRF constant from the original RRF paper. */ +const RRF_K = 60; + +/** + * Merge multiple ranked lists of `MemorySearchResultItem` via Reciprocal Rank + * Fusion. Items appearing in multiple lists get their RRF scores summed. + * + * Returns items sorted by descending RRF score. The `score` field of each + * returned item is replaced by the RRF score for consistent ranking semantics. + */ +function rrfMergeL1(...lists: MemorySearchResultItem[][]): MemorySearchResultItem[] { + const map = new Map(); + + for (const list of lists) { + for (let rank = 0; rank < list.length; rank++) { + const item = list[rank]; + const score = 1 / (RRF_K + rank + 1); + const existing = map.get(item.id); + if (existing) { + existing.rrfScore += score; + } else { + map.set(item.id, { item, rrfScore: score }); + } + } + } + + return [...map.values()] + .sort((a, b) => b.rrfScore - a.rrfScore) + .map(({ item, rrfScore }) => ({ ...item, score: rrfScore })); +} + +// ============================ +// Search implementation +// ============================ + +export async function executeMemorySearch(params: { + query: string; + limit: number; + type?: string; + scene?: string; + vectorStore?: VectorStore; + embeddingService?: EmbeddingService; + logger?: Logger; +}): Promise { + const { + query, + limit, + type: typeFilter, + scene: sceneFilter, + vectorStore, + embeddingService, + logger, + } = params; + + logger?.debug?.( + `${TAG} CALLED: query="${query.slice(0, 100)}", limit=${limit}, ` + + `typeFilter=${typeFilter ?? "(none)"}, sceneFilter=${sceneFilter ?? "(none)"}, ` + + `vectorStore=${vectorStore ? "available" : "UNAVAILABLE"}, ` + + `embeddingService=${embeddingService ? "available" : "UNAVAILABLE"}`, + ); + + if (!query || query.trim().length === 0) { + logger?.debug?.(`${TAG} Empty query, returning empty`); + return { results: [], total: 0, strategy: "none" }; + } + + if (!vectorStore) { + logger?.warn?.(`${TAG} VectorStore not available`); + return { results: [], total: 0, strategy: "none" }; + } + + // ── Determine available capabilities ── + const hasEmbedding = !!embeddingService; + const hasFts = vectorStore.isFtsAvailable(); + + if (!hasEmbedding && !hasFts) { + logger?.warn?.(`${TAG} Neither EmbeddingService nor FTS5 available — cannot search`); + return { + results: [], + total: 0, + strategy: "none", + message: + "Embedding service is not configured and FTS is not available. " + + "Memory search requires an embedding provider or FTS5 support. " + + "Please configure an embedding provider in the embedding.provider setting (e.g. openai_compatible).", + }; + } + + // ── Over-retrieve for later filtering and RRF merging ── + const candidateK = limit * 3; + + // ── Run available search strategies in parallel ── + const [ftsItems, vecItems] = await Promise.all([ + // FTS5 keyword search + (async (): Promise => { + if (!hasFts) return []; + try { + const ftsQuery = buildFtsQuery(query); + if (!ftsQuery) { + logger?.debug?.(`${TAG} [hybrid-fts] No usable FTS tokens from query`); + return []; + } + logger?.debug?.(`${TAG} [hybrid-fts] FTS5 query: "${ftsQuery}"`); + const ftsResults = vectorStore.ftsSearchL1(ftsQuery, candidateK); + logger?.debug?.(`${TAG} [hybrid-fts] FTS5 returned ${ftsResults.length} candidates`); + return ftsResults.map((r) => ({ + id: r.record_id, + content: r.content, + type: r.type, + priority: r.priority, + scene_name: r.scene_name, + score: r.score, + created_at: r.timestamp_start, + updated_at: r.timestamp_end, + })); + } catch (err) { + logger?.warn?.( + `${TAG} [hybrid-fts] FTS5 search failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`, + ); + return []; + } + })(), + + // Vector embedding search + (async (): Promise => { + if (!hasEmbedding) return []; + try { + logger?.debug?.(`${TAG} [hybrid-vec] Generating query embedding...`); + const queryEmbedding = await embeddingService!.embed(query); + logger?.debug?.( + `${TAG} [hybrid-vec] Embedding OK, dims=${queryEmbedding.length}, searching top-${candidateK}...`, + ); + const vecResults: VectorSearchResult[] = vectorStore.search(queryEmbedding, candidateK); + logger?.debug?.(`${TAG} [hybrid-vec] Vector search returned ${vecResults.length} candidates`); + return vecResults.map((r) => ({ + id: r.record_id, + content: r.content, + type: r.type, + priority: r.priority, + scene_name: r.scene_name, + score: r.score, + created_at: r.timestamp_start, + updated_at: r.timestamp_end, + })); + } catch (err) { + logger?.warn?.( + `${TAG} [hybrid-vec] Embedding search failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`, + ); + return []; + } + })(), + ]); + + // ── Determine effective strategy ── + const ftsOk = ftsItems.length > 0; + const vecOk = vecItems.length > 0; + let strategy: string; + + if (ftsOk && vecOk) { + strategy = "hybrid"; + } else if (vecOk) { + strategy = "embedding"; + } else if (ftsOk) { + strategy = "fts"; + } else { + logger?.debug?.(`${TAG} Both search paths returned 0 results`); + return { results: [], total: 0, strategy: hasEmbedding ? "embedding" : "fts" }; + } + + // ── Merge results ── + let results: MemorySearchResultItem[]; + if (strategy === "hybrid") { + results = rrfMergeL1(ftsItems, vecItems); + logger?.debug?.( + `${TAG} [hybrid] RRF merged: fts=${ftsItems.length}, vec=${vecItems.length} → ${results.length} unique`, + ); + } else { + // Single-source: use whichever list has results (already sorted by score) + results = ftsOk ? ftsItems : vecItems; + } + + // ── Apply secondary filters (type, scene) ── + const preFilterCount = results.length; + if (typeFilter) { + results = results.filter((r) => r.type === typeFilter); + logger?.debug?.(`${TAG} After type filter "${typeFilter}": ${results.length}/${preFilterCount}`); + } + if (sceneFilter) { + const normalizedScene = sceneFilter.toLowerCase(); + results = results.filter((r) => + r.scene_name.toLowerCase().includes(normalizedScene), + ); + logger?.debug?.(`${TAG} After scene filter "${sceneFilter}": ${results.length}/${preFilterCount}`); + } + + // ── Trim to requested limit ── + const trimmed = results.slice(0, limit); + + logger?.debug?.( + `${TAG} RESULT (strategy=${strategy}): returning ${trimmed.length} memories ` + + `(scores: [${trimmed.map((r) => r.score.toFixed(3)).join(", ")}])`, + ); + + return { + results: trimmed, + total: trimmed.length, + strategy, + }; +} + +// ============================ +// Tool response formatter +// ============================ + +export function formatSearchResponse(result: MemorySearchResult): string { + if (result.message) { + return result.message; + } + if (result.results.length === 0) { + return "No matching memories found."; + } + + const lines: string[] = [ + `Found ${result.total} matching memories:`, + "", + ]; + + for (const item of result.results) { + const scoreStr = typeof item.score === "number" ? ` (score: ${item.score.toFixed(3)})` : ""; + const sceneStr = item.scene_name ? ` [scene: ${item.scene_name}]` : ""; + const priorityStr = item.priority >= 0 ? ` (priority: ${item.priority})` : " (global instruction)"; + lines.push(`- **[${item.type}]**${priorityStr}${sceneStr}${scoreStr}`); + lines.push(` ${item.content}`); + lines.push(""); + } + + return lines.join("\n"); +} diff --git a/src/utils/backup.ts b/src/utils/backup.ts new file mode 100644 index 0000000..ea05376 --- /dev/null +++ b/src/utils/backup.ts @@ -0,0 +1,156 @@ +/** + * BackupManager: generic file/directory backup utility. + * + * Provides two backup modes: + * - `backupFile(src, category, tag, maxKeep)` — copy a single file + * - `backupDirectory(src, category, tag, maxKeep)` — copy an entire directory + * + * All backups land under `//` with timestamped names. + * After each backup, entries beyond `maxKeep` are automatically pruned + * (oldest first, by lexicographic order on the timestamp-embedded name). + */ + +import fs from "node:fs/promises"; +import path from "node:path"; + +export class BackupManager { + private backupRoot: string; + + /** + * @param backupRoot - Absolute path to the root backup directory + * (e.g. `/.backup`). + */ + constructor(backupRoot: string) { + this.backupRoot = backupRoot; + } + + /** + * Backup a single file. + * + * Destination: `//__.` + * + * @param srcFile - Absolute path to the source file + * @param category - Logical grouping (e.g. "persona") + * @param tag - Additional identifier (e.g. "offset42") + * @param maxKeep - Max backup files to retain in this category (0 = unlimited) + */ + async backupFile( + srcFile: string, + category: string, + tag: string, + maxKeep: number, + ): Promise { + try { + await fs.access(srcFile); + } catch { + return; // Source file doesn't exist, nothing to backup + } + + const destDir = path.join(this.backupRoot, category); + await fs.mkdir(destDir, { recursive: true }); + + const ext = path.extname(srcFile); // e.g. ".md" + const timestamp = formatTimestamp(new Date()); + const destName = `${category}_${timestamp}_${tag}${ext}`; + await fs.copyFile(srcFile, path.join(destDir, destName)); + + if (maxKeep > 0) { + await pruneOldEntries(destDir, maxKeep, "file"); + } + } + + /** + * Backup an entire directory (shallow copy of all files). + * + * Destination: `//__/` + * + * @param srcDir - Absolute path to the source directory + * @param category - Logical grouping (e.g. "scene_blocks") + * @param tag - Additional identifier (e.g. "offset42") + * @param maxKeep - Max backup directories to retain in this category (0 = unlimited) + */ + async backupDirectory( + srcDir: string, + category: string, + tag: string, + maxKeep: number, + ): Promise { + let entries: import("node:fs").Dirent[]; + try { + entries = await fs.readdir(srcDir, { withFileTypes: true }); + } catch { + return; // Source directory doesn't exist + } + + // Only backup regular files (skip subdirectories to avoid EISDIR errors) + const files = entries.filter((e) => e.isFile()).map((e) => e.name); + if (files.length === 0) return; + + const parentDir = path.join(this.backupRoot, category); + const timestamp = formatTimestamp(new Date()); + const destDir = path.join(parentDir, `${category}_${timestamp}_${tag}`); + await fs.mkdir(destDir, { recursive: true }); + + for (const file of files) { + await fs.copyFile(path.join(srcDir, file), path.join(destDir, file)); + } + + if (maxKeep > 0) { + await pruneOldEntries(parentDir, maxKeep, "directory"); + } + } +} + +// ============================ +// Helpers +// ============================ + +function formatTimestamp(d: Date): string { + const pad = (n: number) => String(n).padStart(2, "0"); + return [ + d.getFullYear(), + pad(d.getMonth() + 1), + pad(d.getDate()), + "_", + pad(d.getHours()), + pad(d.getMinutes()), + pad(d.getSeconds()), + ].join(""); +} + +/** + * Keep only the newest `maxKeep` entries in a directory. + * Entries are sorted by name ascending (oldest first) since backup names + * embed timestamps, so lexicographic order = chronological order. + * + * @param dir - Directory containing the backup entries + * @param maxKeep - Number of entries to retain + * @param kind - "file" to unlink, "directory" to rm -rf + */ +async function pruneOldEntries( + dir: string, + maxKeep: number, + kind: "file" | "directory", +): Promise { + let entries: string[]; + try { + entries = await fs.readdir(dir); + } catch { + return; + } + + entries.sort(); // ascending — oldest first + const toRemove = entries.slice(0, Math.max(0, entries.length - maxKeep)); + + for (const name of toRemove) { + try { + if (kind === "file") { + await fs.unlink(path.join(dir, name)); + } else { + await fs.rm(path.join(dir, name), { recursive: true, force: true }); + } + } catch { + // best-effort + } + } +} diff --git a/src/utils/checkpoint.ts b/src/utils/checkpoint.ts new file mode 100644 index 0000000..4da746f --- /dev/null +++ b/src/utils/checkpoint.ts @@ -0,0 +1,543 @@ +/** + * Checkpoint management for tracking memory processing progress. + * + * ## Split-state design + * + * Per-session state is split into two independent namespaces to prevent + * the PipelineManager and L0/L1 runners from overwriting each other's fields: + * + * - **runner_states** (`RunnerSessionState`): owned by CheckpointManager methods + * (markL1*, advanceSession*). Contains L0 capture cursor, L1 cursor, scene name. + * + * - **pipeline_states** (`PipelineSessionState`): owned exclusively by + * PipelineManager via `mergePipelineStates()`. Contains conversation_count, + * extraction times, L2 tracking fields. + * + * Each side only reads/writes its own namespace, eliminating the split-brain + * overwrite bug where pipeline persistStates() could clobber runner-written fields. + * + * ## Concurrency safety + * + * All mutating methods (read-modify-write) are serialized via a per-file async lock. + * Multiple CheckpointManager instances sharing the same file path automatically share + * the same lock, so callers can freely `new CheckpointManager()` without coordination. + * Writes use atomic tmp+rename to prevent corruption on crash. + */ + +import fs from "node:fs/promises"; +import path from "node:path"; +import { randomBytes } from "node:crypto"; + +// ============================ +// Types +// ============================ + +/** + * Per-session state managed by L0/L1 runners (written directly to checkpoint). + * These fields are ONLY written by CheckpointManager methods (markL1*, advanceSession*, etc.) + * and are NEVER touched by the PipelineManager's persistStates(). + */ +export interface RunnerSessionState { + // ═══ L0 — per-session capture cursor ═══ + /** Epoch ms of the newest message captured for THIS session. + * Used instead of the global `Checkpoint.last_captured_timestamp` so that + * concurrent sessions don't advance each other's cursors and cause missed messages. */ + last_captured_timestamp: number; + + // ═══ L1 — cursor & continuity ═══ + /** L0 JSONL cursor: epoch ms of last message processed by L1 */ + last_l1_cursor: number; + /** Last scene name from the most recent L1 extraction (for cross-batch continuity) */ + last_scene_name: string; +} + +/** + * Per-session state managed exclusively by PipelineManager (written via mergePipelineStates). + * These fields are ONLY written by the pipeline's persistStates() callback + * and are NEVER touched by CheckpointManager's L0/L1 methods. + */ +export interface PipelineSessionState { + /** Conversation rounds since last L1 trigger */ + conversation_count: number; + /** ISO timestamp of the last extraction completion */ + last_extraction_time: string; + /** ISO timestamp cursor for incremental extraction reads */ + last_extraction_updated_time: string; + /** Epoch ms of the last notifyConversation call */ + last_active_time: number; + /** Mirrors conversation_count at L1 completion time (for L2 tracking) */ + l2_pending_l1_count: number; + /** + * Current warm-up threshold for L1 triggering. + * Starts at 1 for new sessions and doubles after each L1 completion + * (1 → 2 → 4 → 8 → ...) until it reaches everyNConversations. + * 0 means warm-up is complete (use everyNConversations directly). + */ + warmup_threshold: number; + /** ISO timestamp of last L2 extraction completion */ + l2_last_extraction_time: string; +} + +export interface Checkpoint { + // ═══ Global counters ═══ + /** Epoch ms of the newest message successfully uploaded. Messages with ts > this are new. */ + last_captured_timestamp: number; + /** Total messages processed across all time */ + total_processed: number; + last_persona_at: number; + last_persona_time: string; + request_persona_update: boolean; + persona_update_reason: string; + memories_since_last_persona: number; + scenes_processed: number; + + // ═══ Per-session split state ═══ + /** Runner-managed per-session state (L0 capture cursor, L1 cursor, scene name). + * Written ONLY by CheckpointManager methods. */ + runner_states: Record; + /** Pipeline-managed per-session state (conversation_count, extraction times, etc.). + * Written ONLY by the pipeline's mergePipelineStates(). */ + pipeline_states: Record; + + // ═══ L0 ═══ + /** Total L0 conversation files recorded */ + l0_conversations_count: number; + + // ═══ L1 ═══ + /** Total L1 memories extracted across all time */ + total_memories_extracted: number; +} + +const DEFAULT_RUNNER_STATE: RunnerSessionState = { + last_captured_timestamp: 0, + last_l1_cursor: 0, + last_scene_name: "", +}; + +const DEFAULT_PIPELINE_STATE: PipelineSessionState = { + conversation_count: 0, + last_extraction_time: "", + last_extraction_updated_time: "", + last_active_time: 0, + l2_pending_l1_count: 0, + warmup_threshold: 0, // 0 = graduated (safe default for old sessions missing this field) + l2_last_extraction_time: "", +}; + +const DEFAULT_CHECKPOINT: Checkpoint = { + last_captured_timestamp: 0, + total_processed: 0, + last_persona_at: 0, + last_persona_time: "", + request_persona_update: false, + persona_update_reason: "", + memories_since_last_persona: 0, + scenes_processed: 0, + runner_states: {}, + pipeline_states: {}, + l0_conversations_count: 0, + total_memories_extracted: 0, +}; + +export interface CheckpointLogger { + info(msg: string): void; + warn?(msg: string): void; +} + +const noopLogger: CheckpointLogger = { info() {} }; + +// ============================ +// Per-file async lock +// ============================ +// Keyed by resolved file path. Multiple CheckpointManager instances pointing +// to the same file automatically share the same lock — callers don't need to +// coordinate instance creation. + +const fileLocks = new Map>(); + +/** + * Serialize async critical sections per file path. + * Under no contention the overhead is a single resolved-promise await. + */ +async function withFileLock(filePath: string, fn: () => Promise): Promise { + // Chain after whatever is currently queued for this path + const prev = fileLocks.get(filePath) ?? Promise.resolve(); + let release!: () => void; + const gate = new Promise((r) => { release = r; }); + fileLocks.set(filePath, gate); + + await prev; + try { + return await fn(); + } finally { + release(); + // Clean up the map entry if we're the tail of the chain + if (fileLocks.get(filePath) === gate) { + fileLocks.delete(filePath); + } + } +} + +export class CheckpointManager { + private filePath: string; + private logger: CheckpointLogger; + + constructor(dataDir: string, logger?: CheckpointLogger) { + this.filePath = path.join(dataDir, ".metadata", "recall_checkpoint.json"); + this.logger = logger ?? noopLogger; + } + + // ============================ + // Low-level I/O (internal) + // ============================ + + private async readRaw(): Promise { + try { + const raw = await fs.readFile(this.filePath, "utf-8"); + const parsed = JSON.parse(raw) as Record; + // Merge with defaults for backward compat (old checkpoints lack new fields). + // structuredClone avoids shallow-copy pitfall: without it, the nested + // runner_states/pipeline_states objects in DEFAULT_CHECKPOINT would be + // shared across all callers and mutated in place — corrupting the default. + const cp = { ...structuredClone(DEFAULT_CHECKPOINT), ...parsed } as Checkpoint; + + // Migrate from old session_states format (pre-split) + const oldStates = parsed.session_states as Record> | undefined; + if (oldStates && !parsed.runner_states && !parsed.pipeline_states) { + cp.runner_states = {}; + cp.pipeline_states = {}; + for (const [key, state] of Object.entries(oldStates)) { + cp.runner_states[key] = { + ...DEFAULT_RUNNER_STATE, + last_captured_timestamp: (state.last_captured_timestamp as number) ?? 0, + last_l1_cursor: (state.last_l1_cursor as number) ?? 0, + last_scene_name: (state.last_scene_name as string) ?? "", + }; + cp.pipeline_states[key] = { + ...DEFAULT_PIPELINE_STATE, + conversation_count: (state.conversation_count as number) ?? 0, + last_extraction_time: (state.last_extraction_time as string) ?? "", + last_extraction_updated_time: (state.last_extraction_updated_time as string) ?? "", + last_active_time: (state.last_active_time as number) ?? 0, + l2_pending_l1_count: (state.l2_pending_l1_count as number) ?? 0, + l2_last_extraction_time: (state.l2_last_extraction_time as string) ?? "", + }; + } + } else { + // Ensure per-session states have all fields with defaults + if (cp.runner_states) { + for (const [key, state] of Object.entries(cp.runner_states)) { + cp.runner_states[key] = { ...DEFAULT_RUNNER_STATE, ...state }; + } + } + if (cp.pipeline_states) { + for (const [key, state] of Object.entries(cp.pipeline_states)) { + cp.pipeline_states[key] = { ...DEFAULT_PIPELINE_STATE, ...state }; + } + } + } + return cp; + } catch { + return structuredClone(DEFAULT_CHECKPOINT); + } + } + + /** Atomic write: write to tmp file, then rename into place. */ + private async writeRaw(checkpoint: Checkpoint): Promise { + const dir = path.dirname(this.filePath); + await fs.mkdir(dir, { recursive: true }); + const tmp = `${this.filePath}.tmp.${randomBytes(4).toString("hex")}`; + await fs.writeFile(tmp, JSON.stringify(checkpoint, null, 2), "utf-8"); + await fs.rename(tmp, this.filePath); + } + + // ============================ + // Locked read-modify-write helper + // ============================ + + /** + * Execute a mutating operation under the per-file lock. + * `fn` receives the current checkpoint and may modify it in place; + * the updated checkpoint is atomically written back. + */ + private async mutate(fn: (cp: Checkpoint) => void | Promise): Promise { + return withFileLock(this.filePath, async () => { + const cp = await this.readRaw(); + await fn(cp); + await this.writeRaw(cp); + return cp; + }); + } + + // ============================ + // Public API — read-only + // ============================ + + /** + * Read the current checkpoint (unlocked snapshot). + * + * NOTE: This does NOT acquire the file lock. The returned snapshot may be + * stale if a concurrent `mutate()` is in progress. This is acceptable for + * read-only uses (status display, deciding whether to run a pipeline step). + * + * For read-then-write patterns, always use `mutate()` instead — it acquires + * the lock and re-reads from disk inside the critical section, ensuring the + * update is based on the latest state. + */ + async read(): Promise { + return this.readRaw(); + } + + /** Write a full checkpoint (acquires lock + atomic write). */ + async write(checkpoint: Checkpoint): Promise { + return withFileLock(this.filePath, () => this.writeRaw(checkpoint)); + } + + // ============================ + // Public API — mutating (all serialized via file lock) + // ============================ + + /** + * Advance the captured timestamp after successful upload/recording. + * Also updates total_processed and persona counters. + * + * NOTE: This advances the GLOBAL cursor (`Checkpoint.last_captured_timestamp`). + * For per-session cursor advancement, use `advanceSessionCapturedTimestamp()`. + * The global cursor is kept for aggregate stats / backward compat, but should + * NOT be used as the L0 incremental-capture filter (use per-session instead). + */ + async advanceCapturedTimestamp(maxTimestamp: number, messageCount: number): Promise { + const cp = await this.mutate((cp) => { + cp.last_captured_timestamp = maxTimestamp; + cp.total_processed += messageCount; + cp.memories_since_last_persona += messageCount; + }); + this.logger.info( + `[checkpoint] advanceCapturedTimestamp: -> ${maxTimestamp} (+${messageCount} msgs), ` + + `total_processed=${cp.total_processed}, memories_since_last_persona=${cp.memories_since_last_persona}`, + ); + } + + /** + * Advance the per-session L0 capture cursor after recording messages. + * This is the **primary** cursor for incremental L0 recording — each session + * tracks its own progress independently, preventing cross-session cursor drift. + * + * Also updates the global cursor / total_processed for aggregate stats. + */ + async advanceSessionCapturedTimestamp( + sessionKey: string, + maxTimestamp: number, + messageCount: number, + ): Promise { + const cp = await this.mutate((cp) => { + // Per-session cursor (runner-owned) + const state = this.getRunnerState(cp, sessionKey); + state.last_captured_timestamp = maxTimestamp; + // Global stats (aggregate only — not used for filtering) + cp.last_captured_timestamp = Math.max(cp.last_captured_timestamp, maxTimestamp); + cp.total_processed += messageCount; + cp.memories_since_last_persona += messageCount; + }); + this.logger.info( + `[checkpoint] advanceSessionCapturedTimestamp session=${sessionKey}: -> ${maxTimestamp} ` + + `(+${messageCount} msgs), total_processed=${cp.total_processed}`, + ); + } + + /** + * Increment L0 conversation count. + */ + async incrementL0ConversationCount(): Promise { + await this.mutate((cp) => { + cp.l0_conversations_count += 1; + }); + } + + // ============================ + // Persona methods (L3) + // ============================ + + async markPersonaGenerated(totalProcessed: number): Promise { + await this.mutate((cp) => { + cp.last_persona_at = totalProcessed; + cp.last_persona_time = new Date().toISOString(); + cp.memories_since_last_persona = 0; + cp.request_persona_update = false; + cp.persona_update_reason = ""; + }); + } + + async clearPersonaRequest(): Promise { + await this.mutate((cp) => { + cp.request_persona_update = false; + cp.persona_update_reason = ""; + }); + } + + async setPersonaUpdateRequest(reason: string): Promise { + await this.mutate((cp) => { + cp.request_persona_update = true; + cp.persona_update_reason = reason; + }); + } + + async incrementScenesProcessed(): Promise { + const cp = await this.mutate((cp) => { + cp.scenes_processed += 1; + }); + this.logger.info(`[checkpoint] incrementScenesProcessed: scenes_processed=${cp.scenes_processed}`); + } + + // ============================ + // Per-session helpers — runner state (L0/L1 owned) + // ============================ + + /** + * Get or create runner session state for a session. + */ + getRunnerState(cp: Checkpoint, sessionKey: string): RunnerSessionState { + if (!cp.runner_states) { + cp.runner_states = {}; + } + let state = cp.runner_states[sessionKey]; + if (!state) { + state = { ...DEFAULT_RUNNER_STATE }; + cp.runner_states[sessionKey] = state; + } + return state; + } + + // ============================ + // Per-session helpers — pipeline state (PipelineManager owned) + // ============================ + + /** + * Get or create pipeline session state for a session. + */ + getPipelineState(cp: Checkpoint, sessionKey: string): PipelineSessionState { + if (!cp.pipeline_states) { + cp.pipeline_states = {}; + } + let state = cp.pipeline_states[sessionKey]; + if (!state) { + state = { ...DEFAULT_PIPELINE_STATE, last_active_time: Date.now() }; + cp.pipeline_states[sessionKey] = state; + } + return state; + } + + /** + * Get all pipeline states from checkpoint. + */ + getAllPipelineStates(cp: Checkpoint): Record { + return cp.pipeline_states ?? {}; + } + + /** + * Merge pipeline session states into the checkpoint (used by pipeline persister). + * Acquires the file lock so this is safe against concurrent mutations. + * + * This writes ONLY to `pipeline_states`, never touching `runner_states`. + * This is the core guarantee that eliminates the split-brain overwrite bug. + */ + async mergePipelineStates(states: Record): Promise { + await this.mutate((cp) => { + if (!cp.pipeline_states) cp.pipeline_states = {}; + for (const [key, pState] of Object.entries(states)) { + cp.pipeline_states[key] = { + ...cp.pipeline_states[key], + ...pState, + }; + } + }); + } + + // ============================ + // L1-specific methods + // ============================ + + /** + * Mark L1 extraction completed: reset sinceL1 counter, advance L1 cursor, + * and optionally save the last scene name for cross-batch continuity. + */ + async markL1ExtractionComplete( + sessionKey: string, + memoriesExtracted: number, + cursorTimestamp?: number, + lastSceneName?: string, + ): Promise { + await this.mutate((cp) => { + const state = this.getRunnerState(cp, sessionKey); + if (cursorTimestamp) { + state.last_l1_cursor = cursorTimestamp; + } + if (lastSceneName !== undefined) { + state.last_scene_name = lastSceneName; + } + cp.total_memories_extracted += memoriesExtracted; + cp.memories_since_last_persona += memoriesExtracted; + }); + this.logger.info( + `[checkpoint] markL1ExtractionComplete session=${sessionKey}: ` + + `extracted=${memoriesExtracted}, cursor=${cursorTimestamp ?? "(unchanged)"}, ` + + `lastScene="${lastSceneName ?? "(unchanged)"}"`, + ); + } + + // ============================ + // Atomic capture (race-condition fix) + // ============================ + + /** + * Atomically read the per-session cursor, execute the capture callback, + * and advance the cursor — all within a single file-lock critical section. + * + * This eliminates the race window that existed when `read()` (unlocked) and + * `advanceSessionCapturedTimestamp()` (locked) were separate calls: + * two concurrent `agent_end` events could both read the same stale cursor + * and record duplicate messages. + * + * The callback receives `afterTimestamp` (the current per-session cursor) + * and must return either: + * - `{ maxTimestamp, messageCount }` to advance the cursor, or + * - `null` to leave the cursor unchanged (nothing captured). + * + * L0 conversation count is also incremented inside the lock when messages + * are captured, removing the need for a separate `incrementL0ConversationCount()` call. + * + * @param sessionKey Per-session identifier + * @param pluginStartTimestamp Cold-start floor (used when no cursor exists yet) + * @param fn Async callback that performs the actual capture (recordConversation, etc.) + */ + async captureAtomically( + sessionKey: string, + pluginStartTimestamp: number | undefined, + fn: (afterTimestamp: number) => Promise<{ maxTimestamp: number; messageCount: number } | null>, + ): Promise { + await this.mutate(async (cp) => { + // Read the per-session cursor inside the lock + const state = this.getRunnerState(cp, sessionKey); + let afterTimestamp = state.last_captured_timestamp || 0; + + // Cold-start guard (same logic that was previously in auto-capture.ts) + if (afterTimestamp === 0 && pluginStartTimestamp && pluginStartTimestamp > 0) { + afterTimestamp = pluginStartTimestamp; + } + + const result = await fn(afterTimestamp); + + if (result) { + // Advance per-session cursor (runner-owned) + state.last_captured_timestamp = result.maxTimestamp; + // Global stats (aggregate only — not used for filtering) + cp.last_captured_timestamp = Math.max(cp.last_captured_timestamp, result.maxTimestamp); + cp.total_processed += result.messageCount; + cp.memories_since_last_persona += result.messageCount; + // Increment L0 conversation count (was a separate mutate() call before) + cp.l0_conversations_count += 1; + } + }); + } + +} diff --git a/src/utils/clean-context-runner.ts b/src/utils/clean-context-runner.ts new file mode 100644 index 0000000..e4008d1 --- /dev/null +++ b/src/utils/clean-context-runner.ts @@ -0,0 +1,451 @@ +/** + * CleanContextRunner: executes LLM calls in a fully isolated context + * using runEmbeddedPiAgent (same mechanism as the llm-task extension). + * + * Guarantees: + * 1. Blank conversation history (temporary session file) + * 2. Independent system prompt (only the task prompt) + * 3. No tool calls (tools restricted to minimal read-only set to avoid empty tools[] rejection by some providers) + * 4. No contamination from the main agent's context + */ + +import fs from "node:fs/promises"; +import fsSync from "node:fs"; +import path from "node:path"; +import os from "node:os"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { report } from "../report/reporter.js"; + +/** + * Resolve a preferred temporary directory for memory-tdai operations. + * + * Previously imported from `openclaw/plugin-sdk` as `resolvePreferredOpenClawTmpDir`, + * but that export was removed in openclaw 2026.2.23+. This local implementation + * provides equivalent behavior: + * 1. Try `/tmp/openclaw` (if writable) + * 2. Fall back to `os.tmpdir()/openclaw-` + */ +function resolveOpenClawTmpDir(): string { + const POSIX_DIR = "/tmp/openclaw"; + try { + if (fsSync.existsSync(POSIX_DIR)) { + fsSync.accessSync(POSIX_DIR, fsSync.constants.W_OK | fsSync.constants.X_OK); + return POSIX_DIR; + } + // Try to create it + fsSync.mkdirSync(POSIX_DIR, { recursive: true, mode: 0o700 }); + return POSIX_DIR; + } catch { + // Fall back to os.tmpdir() + const uid = typeof process.getuid === "function" ? process.getuid() : undefined; + const suffix = uid === undefined ? "openclaw" : `openclaw-${uid}`; + const fallback = path.join(os.tmpdir(), suffix); + fsSync.mkdirSync(fallback, { recursive: true }); + return fallback; + } +} + +const TAG = "[memory-tdai] [runner]"; + +interface RunnerLogger { + debug?: (message: string) => void; + info: (message: string) => void; + warn: (message: string) => void; + error: (message: string) => void; +} + +// Dynamic import type — runEmbeddedPiAgent is an internal API +type RunEmbeddedPiAgentFn = (params: Record) => Promise; + +// ── Core import (mirrors voice-call/core-bridge.ts — dist/ only, no jiti) ── + +let _rootCache: string | null = null; + +function findPackageRoot(startDir: string, name: string): string | null { + let dir = startDir; + for (;;) { + const pkgPath = path.join(dir, "package.json"); + try { + if (fsSync.existsSync(pkgPath)) { + const raw = fsSync.readFileSync(pkgPath, "utf8"); + const pkg = JSON.parse(raw) as { name?: string }; + if (pkg.name === name) return dir; + } + } catch { /* ignore */ } + const parent = path.dirname(dir); + if (parent === dir) return null; + dir = parent; + } +} + +function resolveOpenClawRoot(): string { + if (_rootCache) return _rootCache; + const override = process.env.OPENCLAW_ROOT?.trim(); + if (override) { _rootCache = override; return override; } + + const candidates = new Set(); + if (process.argv[1]) candidates.add(path.dirname(process.argv[1])); + candidates.add(process.cwd()); + try { candidates.add(path.dirname(fileURLToPath(import.meta.url))); } catch { /* ignore */ } + + for (const start of candidates) { + const found = findPackageRoot(start, "openclaw"); + if (found) { _rootCache = found; return found; } + } + throw new Error("Unable to resolve OpenClaw root. Set OPENCLAW_ROOT or run `pnpm build`."); +} + +let _loadPromise: Promise | null = null; + +function loadRunEmbeddedPiAgent(logger?: RunnerLogger): Promise { + if (_loadPromise) return _loadPromise; + + _loadPromise = (async () => { + const t0 = Date.now(); + const distPath = path.join(resolveOpenClawRoot(), "dist", "extensionAPI.js"); + if (!fsSync.existsSync(distPath)) { + throw new Error(`Missing core module at ${distPath}. Run \`pnpm build\` or install the official package.`); + } + const mod = await import(pathToFileURL(distPath).href); + if (typeof mod.runEmbeddedPiAgent !== "function") { + throw new Error("runEmbeddedPiAgent not exported from dist/extensionAPI.js"); + } + logger?.info(`${TAG} loadRunEmbeddedPiAgent: dist/ import OK (${Date.now() - t0}ms)`); + return mod.runEmbeddedPiAgent as RunEmbeddedPiAgentFn; + })(); + + _loadPromise.catch(() => { _loadPromise = null; }); + return _loadPromise; +} + +/** + * Pre-warm the embedded agent import. Call this during plugin init to avoid + * the cold-start penalty on the first actual extraction run. + * Returns immediately (fire-and-forget) — errors are swallowed. + */ +export function prewarmEmbeddedAgent(logger?: RunnerLogger): void { + loadRunEmbeddedPiAgent(logger).catch((err) => { + logger?.warn(`${TAG} prewarmEmbeddedAgent: failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`); + }); +} + +function collectText(payloads: Array<{ text?: string; isError?: boolean }> | undefined): string { + const texts = (payloads ?? []) + .filter((p) => !p.isError && typeof p.text === "string") + .map((p) => p.text ?? ""); + return texts.join("\n").trim(); +} + +// ── Model resolution utilities ── + +/** Parsed model reference: { provider, model } */ +export interface ModelRef { + provider: string; + model: string; +} + +/** + * Parse a "provider/model" string into its components. + * Returns undefined if the input is empty or doesn't contain a "/". + * + * Examples: + * "azure/gpt-5.2-chat" → { provider: "azure", model: "gpt-5.2-chat" } + * "custom-host/org/model-v2" → { provider: "custom-host", model: "org/model-v2" } + * "" → undefined + * "bare-model-name" → undefined (no "/" — may be an alias) + */ +export function parseModelRef(raw: string | undefined): ModelRef | undefined { + if (!raw) return undefined; + const trimmed = raw.trim(); + if (!trimmed) return undefined; + + const slashIdx = trimmed.indexOf("/"); + if (slashIdx <= 0 || slashIdx === trimmed.length - 1) return undefined; + + return { + provider: trimmed.slice(0, slashIdx), + model: trimmed.slice(slashIdx + 1), + }; +} + +/** + * Resolve the user's default model from the main OpenClaw config. + * + * Resolution order: + * 1. Read `agents.defaults.model` (string or { primary }) + * 2. If the value contains "/", parse directly + * 3. If not (may be an alias), look up in `agents.defaults.models` alias table + * 4. Return undefined if nothing resolves — let the core use its built-in default + */ +export function resolveModelFromMainConfig(config: unknown): ModelRef | undefined { + if (!config || typeof config !== "object") return undefined; + + const cfg = config as Record; + const agents = cfg.agents as Record | undefined; + if (!agents || typeof agents !== "object") return undefined; + + const defaults = agents.defaults as Record | undefined; + if (!defaults || typeof defaults !== "object") return undefined; + + // Step 1: extract raw model value (string | { primary?: string }) + const modelCfg = defaults.model; + let raw: string | undefined; + if (typeof modelCfg === "string") { + raw = modelCfg.trim(); + } else if (modelCfg && typeof modelCfg === "object") { + const primary = (modelCfg as Record).primary; + raw = typeof primary === "string" ? primary.trim() : undefined; + } + if (!raw) return undefined; + + // Step 2: try direct "provider/model" parse + const direct = parseModelRef(raw); + if (direct) return direct; + + // Step 3: alias lookup — raw doesn't contain "/", check agents.defaults.models + const models = defaults.models as Record | undefined; + if (!models || typeof models !== "object") return undefined; + + const rawLower = raw.toLowerCase(); + for (const [key, entry] of Object.entries(models)) { + if (!entry || typeof entry !== "object") continue; + const alias = (entry as Record).alias; + if (typeof alias !== "string") continue; + if (alias.trim().toLowerCase() !== rawLower) continue; + + // key is "provider/model" format + const resolved = parseModelRef(key); + if (resolved) return resolved; + } + + return undefined; +} + +export interface CleanContextRunnerOptions { + config: unknown; // OpenClawConfig + provider?: string; + model?: string; + /** + * Convenience field: full "provider/model" string. + * Takes precedence over separate `provider`/`model` fields. + * When all three (modelRef, provider, model) are omitted, + * automatically falls back to the main config's `agents.defaults.model`. + */ + modelRef?: string; + /** Allow the LLM to use tools (read_file, write_to_file, etc). Default: false */ + enableTools?: boolean; + /** Logger instance for detailed tracing */ + logger?: RunnerLogger; +} + +// Stable empty directory used as default workspaceDir so that: +// 1. Bootstrap/skills scans find nothing → clean LLM context +// 2. The path is constant → plugin cacheKey stays stable (no re-registration) +let _cleanWorkspaceDir: string | undefined; +async function getCleanWorkspaceDir(): Promise { + if (_cleanWorkspaceDir) return _cleanWorkspaceDir; + const dir = path.join(resolveOpenClawTmpDir(), "memory-tdai-clean-workspace"); + await fs.mkdir(dir, { recursive: true }); + _cleanWorkspaceDir = dir; + return dir; +} + +export class CleanContextRunner { + private options: CleanContextRunnerOptions; + private logger: RunnerLogger | undefined; + /** Resolved provider after modelRef / config fallback */ + private resolvedProvider: string | undefined; + /** Resolved model after modelRef / config fallback */ + private resolvedModel: string | undefined; + + constructor(options: CleanContextRunnerOptions) { + this.options = options; + this.logger = options.logger; + + // Model resolution priority: + // 1. modelRef ("provider/model" string) — highest + // 2. explicit provider + model fields + // 3. main config agents.defaults.model — automatic fallback + // 4. undefined (let core use built-in default) + const fromRef = parseModelRef(options.modelRef); + if (fromRef) { + this.resolvedProvider = fromRef.provider; + this.resolvedModel = fromRef.model; + } else if (options.provider || options.model) { + this.resolvedProvider = options.provider; + this.resolvedModel = options.model; + } else { + // No explicit model specified — fall back to main config + const fromConfig = resolveModelFromMainConfig(options.config); + if (fromConfig) { + this.resolvedProvider = fromConfig.provider; + this.resolvedModel = fromConfig.model; + this.logger?.debug?.( + `${TAG} Using model from main config: ${fromConfig.provider}/${fromConfig.model}`, + ); + } + // else: both undefined → core will use its built-in default (anthropic/claude-opus-4-6) + } + } + + /** + * Run a prompt in a fully isolated clean context. + * Returns the LLM's text output. + * + * When `workspaceDir` is provided it overrides the default `process.cwd()`, + * letting the LLM's file-tool calls resolve paths relative to a custom root. + */ + async run(params: { + prompt: string; + /** Optional system prompt. When provided, `prompt` is used as the user message. */ + systemPrompt?: string; + taskId: string; + timeoutMs?: number; + maxTokens?: number; + workspaceDir?: string; + /** Plugin instance ID for llm_call metric (optional) */ + instanceId?: string; + }): Promise { + const runStartMs = Date.now(); + this.logger?.debug?.(`${TAG} run() start: taskId=${params.taskId}, timeout=${params.timeoutMs ?? 120_000}ms, tools=${this.options.enableTools ? "enabled" : "disabled"}, workspaceDir=${params.workspaceDir ?? "(default)"}`); + + const tmpDir = await fs.mkdtemp( + path.join(resolveOpenClawTmpDir(), `memory-tdai-${params.taskId}-`), + ); + const cleanWorkspace = params.workspaceDir ?? await getCleanWorkspaceDir(); + this.logger?.debug?.(`${TAG} run() tmpDir=${tmpDir}, cleanWorkspace=${cleanWorkspace}`); + + try { + const sessionFile = path.join(tmpDir, "session.json"); + + // Phase 1: Load runEmbeddedPiAgent (fast if dist/ exists or already cached) + const importStartMs = Date.now(); + const runEmbeddedPiAgent = await loadRunEmbeddedPiAgent(this.logger); + const importElapsedMs = Date.now() - importStartMs; + this.logger?.debug?.(`${TAG} run() dynamic import phase: ${importElapsedMs}ms`); + + // Derive a config with plugins disabled to prevent loadOpenClawPlugins + // from re-registering plugins when the workspaceDir differs from the + // gateway's original workspace (cacheKey mismatch triggers full reload). + // + // Security: restrict available tools to the minimal set needed for + // scene extraction (read/write/edit). This prevents the LLM from + // accessing exec, sessions, browser, cron, or any other powerful tools. + // File deletion is handled via "soft-delete" (write empty) + cleanup afterward. + const cleanConfig = { + ...(this.options.config as Record), + plugins: { + ...((this.options.config as Record)?.plugins as Record | undefined), + enabled: false, + }, + tools: { + ...((this.options.config as Record)?.tools as Record | undefined), + // When enableTools=false we still keep one lightweight read-only tool + // so that the tools array sent to the API is non-empty. + // Some providers (e.g. qwencode) reject tools:[] with minItems:1 validation. + allow: this.options.enableTools ? ["read", "write", "edit"] : ["read"], + }, + }; + + // Build the effective prompt: + // If systemPrompt is provided, pass it as a separate parameter to the agent + // and use `prompt` as the user message. Fallback: prepend to prompt if the + // embedded agent doesn't support systemPrompt natively. + const effectivePrompt = params.systemPrompt + ? `${params.systemPrompt}\n\n---\n\n${params.prompt}` + : params.prompt; + + const ts = Date.now(); + const sessionId = `memory-${params.taskId}-session-${ts}`; + const runId = `memory-${params.taskId}-run-${ts}`; + this.logger?.debug?.(`${TAG} run() starting embedded agent: sessionId=${sessionId}, runId=${runId}, provider=${this.resolvedProvider ?? "(default)"}, model=${this.resolvedModel ?? "(default)"}`); + + // Phase 2: Embedded agent run (LLM call + tool calls) + const agentStartMs = Date.now(); + const result = await runEmbeddedPiAgent({ + sessionId, + sessionFile, + workspaceDir: cleanWorkspace, + config: cleanConfig, + prompt: effectivePrompt, + systemPrompt: params.systemPrompt, + timeoutMs: params.timeoutMs ?? 120_000, + runId, + provider: this.resolvedProvider, + model: this.resolvedModel, + // Do NOT pass disableTools:true — that produces tools:[] which some + // providers (qwencode) reject with "[] is too short - 'tools'". + // Instead rely on cleanConfig.tools.allow to restrict the tool set + // to a minimal read-only tool (when enableTools=false). + disableTools: false, + streamParams: { + maxTokens: params.maxTokens, + }, + }); + const agentElapsedMs = Date.now() - agentStartMs; + this.logger?.debug?.(`${TAG} run() embedded agent completed: ${agentElapsedMs}ms`); + + // Phase 3: Collect output + const text = collectText((result as Record).payloads as Array<{ text?: string; isError?: boolean }> | undefined); + const totalMs = Date.now() - runStartMs; + + if (!text) { + // Empty output is normal when the LLM decides there is nothing to + // extract (e.g. trivial greetings). Log a warning instead of + // throwing so the caller can handle it gracefully. + this.logger?.warn?.(`${TAG} run() empty output after ${totalMs}ms (import=${importElapsedMs}ms, agent=${agentElapsedMs}ms) — treating as empty result`); + // llm_call metric (empty output) + if (params.instanceId && this.logger) { + report("llm_call", { + taskId: params.taskId, + provider: this.resolvedProvider ?? "default", + model: this.resolvedModel ?? "default", + inputLength: params.prompt.length, + outputLength: 0, + totalDurationMs: totalMs, + success: true, + error: "empty_output", + }); + } + return ""; + } + + this.logger?.debug?.(`${TAG} run() completed: ${totalMs}ms total (import=${importElapsedMs}ms, agent=${agentElapsedMs}ms), output=${text.length} chars`); + + // ── llm_call metric (success) ── + if (params.instanceId && this.logger) { + report("llm_call", { + taskId: params.taskId, + provider: this.resolvedProvider ?? "default", + model: this.resolvedModel ?? "default", + inputLength: params.prompt.length, + outputLength: text.length, + totalDurationMs: totalMs, + success: true, + error: null, + }); + } + + return text; + } catch (err) { + const totalMs = Date.now() - runStartMs; + this.logger?.error(`${TAG} run() failed after ${totalMs}ms: ${err instanceof Error ? err.stack ?? err.message : String(err)}`); + // ── llm_call metric (failure) ── + if (params.instanceId && this.logger) { + report("llm_call", { + taskId: params.taskId, + provider: this.resolvedProvider ?? "default", + model: this.resolvedModel ?? "default", + inputLength: params.prompt.length, + outputLength: 0, + totalDurationMs: totalMs, + success: false, + error: err instanceof Error ? err.message : String(err), + }); + } + throw err; + } finally { + await fs.rm(tmpDir, { recursive: true, force: true }).catch(() => {}); + } + } +} diff --git a/src/utils/managed-timer.ts b/src/utils/managed-timer.ts new file mode 100644 index 0000000..86a4f97 --- /dev/null +++ b/src/utils/managed-timer.ts @@ -0,0 +1,138 @@ +/** + * ManagedTimer: a named, lifecycle-managed wrapper around setTimeout. + * + * Eliminates repetitive clear→set→fire→clean patterns by providing: + * - `schedule(delayMs, cb)` — cancel any pending timer, set a new one + * - `scheduleAt(epochMs, cb)` — schedule by absolute time point + * - `tryAdvanceTo(epochMs, cb)` — only reschedule if new time is *earlier* + * - `cancel()` — cancel without triggering + * - `flush()` — trigger immediately (for graceful shutdown) + * - `pending` — whether a timer is waiting + * + * The optional `isDestroyed` guard prevents firing after the owner is torn down. + */ + +type TimerHandle = ReturnType; + +export class ManagedTimer { + private handle: TimerHandle | null = null; + private callback: (() => void) | null = null; + /** Absolute epoch-ms when the current timer is scheduled to fire. */ + private scheduledAt = 0; + + constructor( + /** Human-readable name for logging. */ + public readonly name: string, + /** If provided, checked before firing — skips callback when true. */ + private readonly isDestroyed?: () => boolean, + ) {} + + // ── Core operations ────────────────────────────────── + + /** + * Cancel any pending timer and schedule a new one after `delayMs`. + * The callback fires once; the timer auto-clears after firing. + */ + schedule(delayMs: number, callback: () => void): void { + this.cancelInternal(); + this.callback = callback; + this.scheduledAt = Date.now() + delayMs; + this.handle = setTimeout(() => this.fire(), delayMs); + // Don't let pipeline timers keep the process alive in CLI mode. + // In gateway mode the server listener holds the event loop anyway. + this.handle.unref(); + } + + /** + * Cancel any pending timer and schedule to fire at an absolute epoch-ms. + * If `epochMs` is in the past, fires on next tick (delay = 0). + */ + scheduleAt(epochMs: number, callback: () => void): void { + this.cancelInternal(); + this.callback = callback; + this.scheduledAt = epochMs; + const delay = Math.max(0, epochMs - Date.now()); + this.handle = setTimeout(() => this.fire(), delay); + this.handle.unref(); + } + + /** + * Only reschedule if `epochMs` is *earlier* than the current scheduled time. + * This implements the "downward-only" timer pattern (L2 scheduling). + * If no timer is pending, behaves like `scheduleAt()`. + * + * @returns true if the timer was actually advanced (or newly set). + */ + tryAdvanceTo(epochMs: number, callback: () => void): boolean { + if (this.handle === null) { + // No pending timer → set it + this.scheduleAt(epochMs, callback); + return true; + } + + if (epochMs < this.scheduledAt) { + // New time is earlier → reschedule + this.scheduleAt(epochMs, callback); + return true; + } + + // Current timer is already earlier or equal → keep it + return false; + } + + /** + * Cancel the pending timer without triggering the callback. + */ + cancel(): void { + this.cancelInternal(); + } + + /** + * Immediately trigger the callback (if pending) and clear the timer. + * Used for graceful shutdown to flush pending work. + * + * Note: Unlike `fire()`, this method intentionally does NOT check `isDestroyed`. + * This is by design — during shutdown, `destroy()` sets `destroyed = true` first, + * then calls `flush()` to drain pending work. The `isDestroyed` guard only applies + * to natural timer expiration via `fire()`, not to explicit shutdown flushes. + */ + flush(): void { + if (this.handle === null) return; + const cb = this.callback; + this.cancelInternal(); + if (cb) cb(); + } + + // ── Accessors ──────────────────────────────────────── + + /** Whether a timer is currently pending. */ + get pending(): boolean { + return this.handle !== null; + } + + /** The epoch-ms when the current timer is scheduled to fire (0 if none). */ + get scheduledTime(): number { + return this.handle !== null ? this.scheduledAt : 0; + } + + // ── Internals ──────────────────────────────────────── + + private fire(): void { + const cb = this.callback; + this.handle = null; + this.callback = null; + this.scheduledAt = 0; + + if (this.isDestroyed?.()) return; + if (cb) cb(); + } + + private cancelInternal(): void { + if (this.handle !== null) { + clearTimeout(this.handle); + this.handle = null; + } + this.callback = null; + this.scheduledAt = 0; + } +} diff --git a/src/utils/memory-cleaner.ts b/src/utils/memory-cleaner.ts new file mode 100644 index 0000000..24ccf8e --- /dev/null +++ b/src/utils/memory-cleaner.ts @@ -0,0 +1,331 @@ +import fs from "node:fs/promises"; +import path from "node:path"; + +import type { VectorStore } from "../store/vector-store.js"; +import { ManagedTimer } from "./managed-timer.js"; + +interface Logger { + debug?: (message: string) => void; + info: (message: string) => void; + warn: (message: string) => void; + error: (message: string) => void; +} + +export interface MemoryCleanerOptions { + baseDir: string; + retentionDays: number; + cleanTime: string; + logger?: Logger; + vectorStore?: VectorStore; +} + +interface CleanupStats { + scannedFiles: number; + changedFiles: number; + skippedNonShardFiles: number; + deleteFailedFiles: number; +} + +const TAG = "[memory-tdai][cleaner]"; +const L0_DIR_NAME = "conversations"; +const L1_DIR_NAME = "records"; + +export class LocalMemoryCleaner { + private readonly timer: ManagedTimer; + private destroyed = false; + private vectorStore?: VectorStore; + + constructor(private readonly opts: MemoryCleanerOptions) { + this.timer = new ManagedTimer("memory-tdai-cleaner", () => this.destroyed); + this.vectorStore = opts.vectorStore; + } + + setVectorStore(vectorStore: VectorStore | undefined): void { + this.vectorStore = vectorStore; + } + + start(): void { + if (this.destroyed) return; + + const now = new Date(); + const tz = Intl.DateTimeFormat().resolvedOptions().timeZone || "unknown"; + const utcOffset = formatUtcOffset(-now.getTimezoneOffset()); + + this.opts.logger?.info( + `${TAG} Enabled: retentionDays=${this.opts.retentionDays}, cleanTime=${this.opts.cleanTime}, dirs=[${L0_DIR_NAME}, ${L1_DIR_NAME}]`, + ); + this.opts.logger?.info( + `${TAG} Runtime clock: nowLocal=${formatLocalDateTime(now)}, nowIso=${now.toISOString()}, tz=${tz}, utcOffset=${utcOffset}`, + ); + + this.scheduleNext(); + } + + destroy(): void { + if (this.destroyed) return; + this.destroyed = true; + this.timer.cancel(); + this.opts.logger?.info(`${TAG} Stopped`); + } + + async runOnce(nowMs = Date.now()): Promise { + if (this.destroyed) return; + + const retentionDays = this.opts.retentionDays; + if (!(retentionDays > 0)) { + this.opts.logger?.debug?.(`${TAG} Skip run: invalid retentionDays=${retentionDays}`); + return; + } + + // 按“本地自然日”保留策略计算截止时间。 + // 例如 retentionDays=2，今天是 03-15，则保留 03-14/03-15，删除早于 03-14 00:00:00.000 的记录。 + const cutoffMs = computeCutoffMsByLocalDay(nowMs, retentionDays); + const targetDirs = [ + path.join(this.opts.baseDir, L0_DIR_NAME), + path.join(this.opts.baseDir, L1_DIR_NAME), + ]; + + const total: CleanupStats = { + scannedFiles: 0, + changedFiles: 0, + skippedNonShardFiles: 0, + deleteFailedFiles: 0, + }; + + for (const dirPath of targetDirs) { + const stats = await this.cleanDirectory(dirPath, cutoffMs); + total.scannedFiles += stats.scannedFiles; + total.changedFiles += stats.changedFiles; + total.skippedNonShardFiles += stats.skippedNonShardFiles; + total.deleteFailedFiles += stats.deleteFailedFiles; + } + + if (this.vectorStore) { + const vectorStore = this.vectorStore; + const cutoffIso = new Date(cutoffMs).toISOString(); + + let removedL0 = 0; + let removedL1 = 0; + let failedL0DbCleanup = 0; + let failedL1DbCleanup = 0; + + try { + removedL0 = vectorStore.deleteL0ExpiredByRecordedAt(cutoffIso); + } catch (err) { + failedL0DbCleanup = 1; + this.opts.logger?.warn( + `${TAG} SQLite cleanup L0 failed: ${err instanceof Error ? err.message : String(err)}`, + ); + } + + try { + removedL1 = vectorStore.deleteL1ExpiredByUpdatedTime(cutoffIso); + } catch (err) { + failedL1DbCleanup = 1; + this.opts.logger?.warn( + `${TAG} SQLite cleanup L1 failed: ${err instanceof Error ? err.message : String(err)}`, + ); + } + + if (removedL1 > 0 || removedL0 > 0) { + total.changedFiles += 1; + } + + this.opts.logger?.info( + `${TAG} SQLite cleanup done: removedL1Records=${removedL1}, removedL0Records=${removedL0}, failedL1DbCleanup=${failedL1DbCleanup}, failedL0DbCleanup=${failedL0DbCleanup}, cutoffIso=${cutoffIso}`, + ); + } + + this.opts.logger?.info( + `${TAG} Cleanup done: scannedFiles=${total.scannedFiles}, changedFiles=${total.changedFiles}, skippedNonShardFiles=${total.skippedNonShardFiles}, deleteFailedFiles=${total.deleteFailedFiles}`, + ); + + } + + private scheduleNext(): void { + const nowMs = Date.now(); + const now = new Date(nowMs); + const next = nextRunAt(this.opts.cleanTime, nowMs); + const targetToday = buildTodayRunTime(this.opts.cleanTime, nowMs); + const passedToday = targetToday <= nowMs; + const delayMs = Math.max(0, next - nowMs); + + this.opts.logger?.info( + `${TAG} Schedule next run: nowLocal=${formatLocalDateTime(now)}, cleanTime=${this.opts.cleanTime}, targetTodayLocal=${formatLocalDateTime(new Date(targetToday))}, passedToday=${passedToday}, nextRunLocal=${formatLocalDateTime(new Date(next))}, nextRunIso=${new Date(next).toISOString()}, delayMs=${delayMs}`, + ); + + this.timer.scheduleAt(next, () => { + const firedAtMs = Date.now(); + this.opts.logger?.info( + `${TAG} Timer fired: scheduledLocal=${formatLocalDateTime(new Date(next))}, firedLocal=${formatLocalDateTime(new Date(firedAtMs))}, driftMs=${firedAtMs - next}`, + ); + void this.runAndReschedule(); + }); + } + + private async runAndReschedule(): Promise { + if (this.destroyed) return; + const runStart = new Date(); + this.opts.logger?.info( + `${TAG} Cleanup tick start: nowLocal=${formatLocalDateTime(runStart)}, nowIso=${runStart.toISOString()}`, + ); + + try { + await this.runOnce(); + } catch (err) { + this.opts.logger?.error(`${TAG} Cleanup failed: ${err instanceof Error ? err.stack ?? err.message : String(err)}`); + } finally { + if (!this.destroyed) { + this.scheduleNext(); + } + } + } + + private async cleanDirectory(dirPath: string, cutoffMs: number): Promise { + const stats: CleanupStats = { + scannedFiles: 0, + changedFiles: 0, + skippedNonShardFiles: 0, + deleteFailedFiles: 0, + }; + + let entries; + try { + entries = await fs.readdir(dirPath, { withFileTypes: true }); + } catch { + + this.opts.logger?.debug?.(`${TAG} Directory not found, skip: ${dirPath}`); + return stats; + } + + for (const entry of entries) { + if (!entry.isFile()) continue; + if (!isJsonLikeFile(entry.name)) continue; + + const filePath = path.join(dirPath, entry.name); + stats.scannedFiles += 1; + + // 仅支持日期分片文件：YYYY-MM-DD(.jsonl/.json) + const shard = extractShardDateFromFileName(entry.name); + if (!shard) { + stats.skippedNonShardFiles += 1; + this.opts.logger?.debug?.(`${TAG} Skip non-shard file: ${filePath}`); + continue; + } + + const dayEndMs = localDayEndMs(shard.year, shard.month, shard.day); + if (dayEndMs < cutoffMs) { + try { + await fs.unlink(filePath); + stats.changedFiles += 1; + this.opts.logger?.info(`${TAG} Removed expired file by name: ${filePath}`); + } catch (err) { + stats.deleteFailedFiles += 1; + this.opts.logger?.warn( + `${TAG} Failed to delete expired shard file ${filePath}: ${err instanceof Error ? err.message : String(err)}`, + ); + } + } else { + this.opts.logger?.debug?.(`${TAG} Keep shard file by name: ${filePath}`); + } + } + + return stats; + } +} + +function isJsonLikeFile(name: string): boolean { + return name.endsWith(".jsonl") || name.endsWith(".json"); +} + +function extractShardDateFromFileName( + fileName: string, +): { year: number; month: number; day: number } | undefined { + + // Supported format: YYYY-MM-DD.jsonl | YYYY-MM-DD.json + const m = /^(\d{4})-(\d{2})-(\d{2})\.(?:jsonl|json)$/.exec(fileName); + if (!m) return undefined; + + const year = Number(m[1]); + const month = Number(m[2]); + const day = Number(m[3]); + + if (!Number.isInteger(year) || !Number.isInteger(month) || !Number.isInteger(day)) { + return undefined; + } + + if (month < 1 || month > 12 || day < 1 || day > 31) { + return undefined; + } + + const probe = new Date(year, month - 1, day); + if ( + probe.getFullYear() !== year + || probe.getMonth() !== month - 1 + || probe.getDate() !== day + ) { + return undefined; + } + + return { year, month, day }; +} + +function localDayEndMs(year: number, month: number, day: number): number { + const end = new Date(year, month - 1, day, 23, 59, 59, 999); + return end.getTime(); +} + +function formatLocalDateTime(d: Date): string { + const y = d.getFullYear(); + const m = String(d.getMonth() + 1).padStart(2, "0"); + const day = String(d.getDate()).padStart(2, "0"); + const hh = String(d.getHours()).padStart(2, "0"); + const mm = String(d.getMinutes()).padStart(2, "0"); + const ss = String(d.getSeconds()).padStart(2, "0"); + return `${y}-${m}-${day} ${hh}:${mm}:${ss}`; +} + +function formatUtcOffset(offsetMinutes: number): string { + const sign = offsetMinutes >= 0 ? "+" : "-"; + const abs = Math.abs(offsetMinutes); + const hh = String(Math.floor(abs / 60)).padStart(2, "0"); + const mm = String(abs % 60).padStart(2, "0"); + return `${sign}${hh}:${mm}`; +} + +function computeCutoffMsByLocalDay(nowMs: number, retentionDays: number): number { + // 自然日策略，保留“今天 + 往前 retentionDays-1 天” + // 删除阈值为 keepStart 当天 00:00:00.000（本地时区） + const now = new Date(nowMs); + const keepStart = new Date(now.getFullYear(), now.getMonth(), now.getDate(), 0, 0, 0, 0); + keepStart.setDate(keepStart.getDate() - (retentionDays - 1)); + return keepStart.getTime(); +} + +function buildTodayRunTime(cleanTime: string, nowMs: number): number { + + const [hRaw, mRaw] = cleanTime.split(":"); + const hour = Number(hRaw); + const minute = Number(mRaw); + + const target = new Date(nowMs); + target.setHours(hour, minute, 0, 0); + return target.getTime(); +} + +function nextRunAt(cleanTime: string, nowMs: number): number { + + const [hRaw, mRaw] = cleanTime.split(":"); + const hour = Number(hRaw); + const minute = Number(mRaw); + + const now = new Date(nowMs); + const next = new Date(nowMs); + next.setHours(hour, minute, 0, 0); + + if (next.getTime() <= now.getTime()) { + next.setDate(next.getDate() + 1); + } + + return next.getTime(); +} diff --git a/src/utils/pipeline-manager.ts b/src/utils/pipeline-manager.ts new file mode 100644 index 0000000..01c47c9 --- /dev/null +++ b/src/utils/pipeline-manager.ts @@ -0,0 +1,1087 @@ +/** + * MemoryPipelineManager: manages the L0→L1→L2→L3 memory extraction pipeline. + * + * ## Layered architecture + * + * - **L0 (capture)**: `auto-capture.ts` extracts new messages from each + * `agent_end` event, sanitizes them, and passes them to the pipeline via + * `notifyConversation(sessionKey, messages)`. Messages are buffered + * locally per-session — NO remote call happens at this stage. + * + * - **L1 (batch extraction / ingest)**: When the conversation count reaches + * `everyNConversations` OR the session goes idle for `l1IdleTimeoutSeconds`, + * the L1 Runner is invoked with all buffered messages. The runner receives + * `{ sessionKey, msg, bg_msg }` and is responsible for ingesting/extracting + * them (e.g. calling appendEvent, or running local extraction logic). + * `bg_msg` is reserved for background context; currently always empty. + * + * - **L2 (scene extraction)**: Per-session downward-only timer. After each + * L2 completion, the next fire time is set to `now + maxInterval`. When + * L1 completes (new memory event), the fire time is advanced (but never + * postponed) to `max(now + delay, lastL2 + minInterval)`. When the timer + * fires, if the session is cold (inactive > `sessionActiveWindowHours`), + * the timer is cancelled rather than triggering L2 — it will be re-armed + * by the next L1 event. + * + * - **L3 (persona generation)**: Global mutex (concurrency=1) + pending flag + * dedup. Triggered after L2 completes. + * + * ## Timer semantics + * + * L1 uses a **resettable timer** (classic idle/debounce): each conversation + * resets the countdown to `l1IdleTimeoutSeconds`. When the timer fires, + * buffered messages are flushed through L1. + * + * L2 uses a **downward-only timer**: the scheduled fire time can only be + * moved earlier, never later. This ensures both the maxInterval guarantee + * and the delay-after-L1 responsiveness, while minInterval acts as a floor. + * + * Both timer types are implemented via `ManagedTimer` to eliminate + * repetitive clear→set→fire→clean boilerplate. + * + * ## Trigger paths for L1 + * A. **Conversation threshold** (primary): when `conversation_count >= + * effectiveThreshold` in `notifyConversation()`, L1 is triggered + * immediately with all buffered messages. The effective threshold + * is influenced by warm-up mode (see below). + * B. **Idle timeout** (catch-up): when a session goes idle for + * `l1IdleTimeoutSeconds`, L1 fires with whatever messages have + * been buffered (below threshold). + * C. **Shutdown flush**: on graceful shutdown, all pending buffers + * are flushed through L1 then L2. + * + * ## Warm-up mode + * + * When `enableWarmup` is true (default), new sessions use an exponentially + * increasing L1 trigger threshold instead of jumping straight to + * `everyNConversations`. The sequence is: 1 → 2 → 4 → 8 → ... → + * everyNConversations. This ensures early conversations are processed + * quickly (first conversation triggers L1 immediately), while gradually + * reducing processing frequency as the session matures. + * + * The `warmup_threshold` field in PipelineSessionState tracks the current + * threshold. A value of 0 means warm-up is complete (graduated to + * steady-state). The threshold doubles after each successful L1 run. + * + * ## Trigger paths for L2 + * A. **Delay-after-L1**: L1 completes → timer advanced to + * `max(now + delay, lastL2 + min)` → fires → enqueue L2. + * B. **MaxInterval guarantee**: L2 completes → timer set to + * `now + maxInterval` → fires → enqueue L2 (if session active). + * C. **Shutdown flush**: all pending L2 timers are flushed. + * + * All queues use SerialQueue (concurrency=1) for serial execution. + * + * ## Design doc + * See `docs/08-pipeline-refactor-design.md` for full architecture. + */ + +import type { PipelineSessionState } from "./checkpoint.js"; +import { SessionFilter } from "./session-filter.js"; +import { ManagedTimer } from "./managed-timer.js"; +import { SerialQueue } from "./serial-queue.js"; +import { report } from "../report/reporter.js"; + +// ============================ +// Types +// ============================ + +interface Logger { + debug?: (message: string) => void; + info: (message: string) => void; + warn: (message: string) => void; + error: (message: string) => void; +} + +/** A single captured message ready for L1 processing. */ +export interface CapturedMessage { + role: "user" | "assistant" | "tool"; + content: string; + /** ISO timestamp string */ + timestamp: string; +} + +/** Pipeline configuration — all time values in seconds. */ +export interface PipelineConfig { + /** + * Conversation count threshold to trigger L1 batch processing. + * When a session's conversation_count reaches this value, + * L1 is triggered immediately with all buffered messages. + * Default: 5. + */ + everyNConversations: number; + + /** + * Enable warm-up mode for new sessions. + * When enabled, the L1 trigger threshold starts at 1 and doubles after + * each successful L1 run (1 → 2 → 4 → 8 → ... → everyNConversations), + * allowing early sessions to be processed more aggressively. + * Default: true. + */ + enableWarmup: boolean; + + l1: { + /** Idle timeout before triggering L1 (seconds, default: 60) */ + idleTimeoutSeconds: number; + }; + + l2: { + /** + * Delay after L1 completes before triggering L2 (seconds, default: 90). + * Allows remote L1 to finish generating records asynchronously. + */ + delayAfterL1Seconds: number; + /** Minimum interval between L2 extractions per session (seconds, default: 300) */ + minIntervalSeconds: number; + /** + * Maximum interval between L2 extractions per session (seconds, default: 1800). + * Even without new L1 completions, L2 will poll at this interval for active sessions. + */ + maxIntervalSeconds: number; + /** + * Sessions inactive longer than this (hours, default: 24) stop L2 polling. + * Prevents wasting resources on abandoned sessions. + */ + sessionActiveWindowHours: number; + }; +} + +/** Result returned by the L1 runner. */ +export interface L1RunnerResult { + /** Number of messages successfully processed */ + processedCount?: number; +} + +/** L1 runner — batch-processes buffered messages for a session. */ +export type L1Runner = (params: { + sessionKey: string; + msg: CapturedMessage[]; + bg_msg: CapturedMessage[]; +}) => Promise; + +/** Result returned by the L2 extraction runner. */ +export interface L2RunnerResult { + /** The latest `updated_at` cursor from the processed batch. */ + latestCursor?: string; +} + +/** L2 extraction runner — processes a single session's records. */ +export type L2Runner = (sessionKey: string, cursor?: string) => Promise; + +/** L3 runner — generates persona from all sessions' scene data. */ +export type L3Runner = () => Promise; + +/** Callback to persist session states to checkpoint. */ +export type PipelineStatePersister = (states: Record) => Promise; + +const TAG = "[memory-tdai] [pipeline]"; + +// ============================ +// Per-session timer state (in memory only) +// ============================ + +interface SessionTimerState { + /** L1 idle timer (resettable): debounces conversation activity. */ + l1Idle: ManagedTimer; + /** L2 schedule timer (downward-only): next L2 fire time, only moves earlier. */ + l2Schedule: ManagedTimer; + /** Whether an L1 task is already queued or running for this session. */ + l1Queued: boolean; + /** Whether an L2 task is already queued or running for this session. */ + l2Queued: boolean; + /** Consecutive L1 failure count for retry limiting. Reset on success or new conversation. */ + l1RetryCount: number; +} + +export class MemoryPipelineManager { + // Config (converted to ms internally) + private readonly l1IdleTimeoutMs: number; + private readonly everyNConversations: number; + private readonly enableWarmup: boolean; + private readonly l2DelayAfterL1Ms: number; + private readonly l2MinIntervalMs: number; + private readonly l2MaxIntervalMs: number; + private readonly sessionActiveWindowMs: number; + + /** Delay before retrying a failed L1 (ms). */ + private readonly L1_RETRY_DELAY_MS = 30_000; // 30 seconds + /** Max consecutive L1 retries per session before giving up. */ + private readonly L1_MAX_RETRIES = 5; + + // Queues (named for diagnostics) + private readonly l1Queue = new SerialQueue("L1"); + private readonly l2Queue = new SerialQueue("L2"); + private readonly l3Queue = new SerialQueue("L3"); + + // L3 dedup flag + private l3Pending = false; + private l3Running = false; + + // Per-session state + private readonly sessionStates = new Map(); + private readonly sessionTimers = new Map(); + + // Per-session message buffer: messages accumulated since last L1 run + private readonly messageBuffers = new Map(); + + // Per-session L2 last run time (epoch ms, for minInterval floor) + private readonly l2LastRunTime = new Map(); + + // Callbacks + private l1Runner: L1Runner | null = null; + private l2Runner: L2Runner | null = null; + private l3Runner: L3Runner | null = null; + private persister: PipelineStatePersister | null = null; + private logger: Logger | undefined; + + // Unified session filter (internal sessions + excludeAgents) + private readonly sessionFilter: SessionFilter; + + // Lifecycle + private destroyed = false; + + /** Plugin instance ID for metric reporting (set externally after async init). */ + instanceId?: string; + + // Session GC: runs periodically to evict cold sessions from memory + /** Multiplier on sessionActiveWindowMs to determine GC eligibility. */ + private readonly SESSION_GC_INACTIVE_MULTIPLIER = 3; + /** Run GC every N calls to notifyConversation. */ + private readonly SESSION_GC_EVERY_N_NOTIFICATIONS = 50; + /** Counter for GC scheduling. */ + private notifyCounter = 0; + + constructor(config: PipelineConfig, logger?: Logger, sessionFilter?: SessionFilter) { + this.l1IdleTimeoutMs = config.l1.idleTimeoutSeconds * 1000; + this.everyNConversations = config.everyNConversations; + this.enableWarmup = config.enableWarmup; + this.l2DelayAfterL1Ms = config.l2.delayAfterL1Seconds * 1000; + this.l2MinIntervalMs = config.l2.minIntervalSeconds * 1000; + this.l2MaxIntervalMs = config.l2.maxIntervalSeconds * 1000; + this.sessionActiveWindowMs = config.l2.sessionActiveWindowHours * 60 * 60 * 1000; + this.logger = logger; + this.sessionFilter = sessionFilter ?? new SessionFilter(); + + this.logger?.info( + `${TAG} Initialized: everyNConversations=${config.everyNConversations}, ` + + `warmup=${config.enableWarmup ? "enabled" : "disabled"}, ` + + `l1IdleTimeout=${config.l1.idleTimeoutSeconds}s, ` + + `l2DelayAfterL1=${config.l2.delayAfterL1Seconds}s, ` + + `l2MinInterval=${config.l2.minIntervalSeconds}s, ` + + `l2MaxInterval=${config.l2.maxIntervalSeconds}s, ` + + `sessionActiveWindow=${config.l2.sessionActiveWindowHours}h`, + ); + + // Wire up queue debug logging + if (this.logger?.debug) { + const debugFn = (msg: string) => this.logger?.debug?.(`${TAG} ${msg}`); + this.l1Queue.setDebugLogger(debugFn); + this.l2Queue.setDebugLogger(debugFn); + this.l3Queue.setDebugLogger(debugFn); + } + } + + // ============================ + // Setup + // ============================ + + setL1Runner(runner: L1Runner): void { + this.l1Runner = runner; + } + + setL2Runner(runner: L2Runner): void { + this.l2Runner = runner; + } + + setL3Runner(runner: L3Runner): void { + this.l3Runner = runner; + } + + setPersister(persister: PipelineStatePersister): void { + this.persister = persister; + } + + /** + * Restore session states from checkpoint and start the pipeline. + * Sessions with pending counts will be immediately re-enqueued. + */ + start(restoredStates?: Record): void { + if (this.destroyed) return; + + if (restoredStates) { + let skipped = 0; + for (const [sessionKey, state] of Object.entries(restoredStates)) { + if (this.sessionFilter.shouldSkip(sessionKey)) { + skipped++; + continue; + } + // Backfill warmup_threshold for sessions persisted before warm-up feature. + // Missing field → treat as graduated (warmup already complete). + const patched = { ...state }; + if (patched.warmup_threshold == null) { + patched.warmup_threshold = 0; + } + this.sessionStates.set(sessionKey, patched); + } + this.logger?.info( + `${TAG} Restored ${this.sessionStates.size} session state(s) from checkpoint` + + (skipped > 0 ? ` (filtered ${skipped} internal)` : ""), + ); + } + + // Recovery: re-enqueue sessions with pending work + this.recoverPendingSessions(); + + this.logger?.info(`${TAG} Pipeline started`); + } + + // ============================ + // L0→L1: Notify (called from auto-capture on agent_end) + // ============================ + + /** + * Get the effective conversation threshold for a session, considering warm-up. + * + * When warm-up is enabled, new sessions start with threshold=1 and double + * after each successful L1 run: 1 → 2 → 4 → 8 → ... → everyNConversations. + * Once the threshold reaches everyNConversations, warm-up is considered complete + * (warmup_threshold is set to 0) and the fixed config value is used. + */ + private getEffectiveThreshold(state: PipelineSessionState): number { + if (!this.enableWarmup) return this.everyNConversations; + // warmup_threshold === 0 means warm-up completed; use steady-state config + if (state.warmup_threshold <= 0) return this.everyNConversations; + return Math.min(state.warmup_threshold, this.everyNConversations); + } + + /** + * Advance the warm-up threshold for a session after a successful L1 run. + * Doubles the threshold until it reaches everyNConversations, then marks + * warm-up as complete (warmup_threshold = 0). + */ + private advanceWarmupThreshold(state: PipelineSessionState): void { + if (!this.enableWarmup) return; + if (state.warmup_threshold <= 0) return; // already graduated + + const next = state.warmup_threshold * 2; + if (next >= this.everyNConversations) { + // Graduated: switch to steady-state + state.warmup_threshold = 0; + this.logger?.debug?.(`${TAG} Warm-up graduated → using steady-state threshold ${this.everyNConversations}`); + } else { + state.warmup_threshold = next; + this.logger?.debug?.(`${TAG} Warm-up advanced → next threshold ${next}`); + } + } + + /** + * Notify the pipeline that a conversation round has ended for a session, + * and buffer the captured messages for L1 batch processing. + * + * Two trigger paths start here: + * - **Path A (threshold)**: if conversation_count >= effective threshold + * (warm-up or steady-state), trigger L1 immediately with all buffered messages. + * - **Path B (idle)**: reset the L1 idle timer. When the timer fires (user + * stops chatting), L1 runs with whatever has been buffered. + */ + async notifyConversation(sessionKey: string, messages: CapturedMessage[]): Promise { + if (this.destroyed) return; + if (this.sessionFilter.shouldSkip(sessionKey)) return; + + const state = this.getOrCreateState(sessionKey); + state.conversation_count += 1; + state.last_active_time = Date.now(); + + // Reset L1 retry count on new conversation (environment may have recovered) + const timers = this.getOrCreateTimers(sessionKey); + timers.l1RetryCount = 0; + + // Buffer messages for L1 + const buffer = this.messageBuffers.get(sessionKey) ?? []; + buffer.push(...messages); + this.messageBuffers.set(sessionKey, buffer); + + const effectiveThreshold = this.getEffectiveThreshold(state); + const warmupInfo = this.enableWarmup && state.warmup_threshold > 0 + ? ` (warmup: ${state.warmup_threshold})` + : ""; + + this.logger?.debug?.( + `${TAG} [${sessionKey}] notify: conversation_count=${state.conversation_count}/${effectiveThreshold}${warmupInfo}, ` + + `buffered_messages=${buffer.length} (+${messages.length} new)`, + ); + + await this.persistStates(); + + // Path A: conversation count reached effective threshold → trigger L1 batch + if (state.conversation_count >= effectiveThreshold) { + this.logger?.debug?.( + `${TAG} [${sessionKey}] Conversation threshold reached (${state.conversation_count}>=${effectiveThreshold}${warmupInfo}), triggering L1`, + ); + this.enqueueL1(sessionKey); + return; // skip idle timer reset — L1 is already triggered + } + + // Path B: below threshold → reset L1 idle timer (catch residual later) + timers.l1Idle.schedule(this.l1IdleTimeoutMs, () => this.onL1IdleTimeout(sessionKey)); + this.logger?.debug?.( + `${TAG} [${sessionKey}] L1 idle timer reset (${this.l1IdleTimeoutMs / 1000}s)`, + ); + + // Periodic GC: evict cold sessions from memory + this.notifyCounter += 1; + if (this.notifyCounter >= this.SESSION_GC_EVERY_N_NOTIFICATIONS) { + this.notifyCounter = 0; + this.gcStaleSessions(); + } + } + + // ============================ + // Graceful shutdown + // ============================ + + /** + * Maximum time (ms) to wait for pipeline flush during destroy. + * Must be shorter than the gateway_stop hook timeout (3 s) to leave + * headroom for VectorStore / EmbeddingService cleanup that runs after. + */ + private readonly DESTROY_TIMEOUT_MS = 2_000; + + /** + * Graceful shutdown with timeout protection: + * 1. Mark destroyed, stop accepting new work + * 2. Attempt to flush pending L1/L2/L3 work within DESTROY_TIMEOUT_MS + * 3. If flush times out or fails, persist current state for recovery on next startup + * 4. Pending work is never lost — it will be recovered via checkpoint on next start() + */ + async destroy(): Promise { + if (this.destroyed) return; + this.destroyed = true; + + this.logger?.info( + `${TAG} Destroying pipeline (timeout=${this.DESTROY_TIMEOUT_MS}ms)...`, + ); + + try { + let timeoutId: ReturnType | undefined; + await Promise.race([ + this._doFlush(), + new Promise((_, reject) => { + timeoutId = setTimeout(() => reject(new Error("destroy timeout")), this.DESTROY_TIMEOUT_MS); + }), + ]).finally(() => { + if (timeoutId !== undefined) clearTimeout(timeoutId); + }); + this.logger?.info(`${TAG} Pipeline flushed successfully`); + } catch (err) { + this.logger?.warn( + `${TAG} Pipeline flush timed out or failed: ${err instanceof Error ? err.message : String(err)}. ` + + `Pending work will be recovered on next startup.`, + ); + } + + // Always persist state — whether flush succeeded, timed out, or failed. + // This ensures pending work (buffered messages, L2 pending counts) is + // saved to checkpoint and can be recovered by recoverPendingSessions(). + try { + await this.persistStates(); + } catch (err) { + this.logger?.error( + `${TAG} Failed to persist states during destroy: ${err instanceof Error ? err.message : String(err)}`, + ); + } + + this.logger?.info(`${TAG} Pipeline destroyed`); + } + + /** + * Internal: attempt to flush all pending pipeline work (L1 → L2 → L3). + * Extracted from destroy() so it can be wrapped with a timeout. + */ + private async _doFlush(): Promise { + // Step 1: Flush all L1 idle timers — only enqueue if there are buffered messages + for (const [sessionKey, timers] of this.sessionTimers) { + if (timers.l1Idle.pending) { + timers.l1Idle.cancel(); // don't fire the idle callback directly + const buffer = this.messageBuffers.get(sessionKey); + if (buffer && buffer.length > 0) { + this.logger?.debug?.(`${TAG} [${sessionKey}] Flush: enqueuing L1 for ${buffer.length} buffered messages`); + this.enqueueL1(sessionKey, "flush"); + } + } + } + + // Step 2: Wait for L1 queue to drain + this.logger?.debug?.(`${TAG} Waiting for L1 queue to drain (size=${this.l1Queue.size})`); + await this.l1Queue.onIdle(); + + // Step 3: Flush all L2 schedule timers + for (const [sessionKey, timers] of this.sessionTimers) { + if (timers.l2Schedule.pending) { + this.logger?.debug?.(`${TAG} [${sessionKey}] Flush: triggering L2 schedule timer`); + timers.l2Schedule.flush(); + } + } + + // Step 4: Wait for all remaining queues to drain + this.logger?.debug?.(`${TAG} Waiting for queues to drain (l2=${this.l2Queue.size}, l3=${this.l3Queue.size})`); + await Promise.all([ + this.l2Queue.onIdle(), + this.l3Queue.onIdle(), + ]); + } + + // ============================ + // Internal: L1 idle timeout handler + // ============================ + + private onL1IdleTimeout(sessionKey: string): void { + const buffer = this.messageBuffers.get(sessionKey); + const state = this.sessionStates.get(sessionKey); + + if ((!buffer || buffer.length === 0) && (!state || state.conversation_count === 0)) { + this.logger?.debug?.( + `${TAG} [${sessionKey}] L1 idle timeout but no pending messages or conversations`, + ); + return; + } + + this.logger?.debug?.( + `${TAG} [${sessionKey}] L1 idle timeout fired (buffered=${buffer?.length ?? 0}, conversations=${state?.conversation_count ?? 0})`, + ); + this.enqueueL1(sessionKey, "idle_timeout"); + } + + // ============================ + // Internal: L1 queue + // ============================ + + private enqueueL1(sessionKey: string, triggerReason: "threshold" | "idle_timeout" | "flush" = "threshold"): void { + const timers = this.getOrCreateTimers(sessionKey); + + // Don't double-queue + if (timers.l1Queued) { + this.logger?.debug?.(`${TAG} [${sessionKey}] L1 already queued, skipping`); + return; + } + + // Cancel idle timer if running (threshold beat it) + timers.l1Idle.cancel(); + + timers.l1Queued = true; + this.logger?.debug?.(`${TAG} [${sessionKey}] Enqueuing L1 (queue=${this.l1Queue.name})`); + + // ── pipeline_l1_trigger metric ── + const state = this.sessionStates.get(sessionKey); + const buffer = this.messageBuffers.get(sessionKey); + if (this.instanceId && this.logger) { + report("pipeline_l1_trigger", { + sessionKey, + triggerReason, + conversationCount: state?.conversation_count ?? 0, + bufferedMessageCount: buffer?.length ?? 0, + }); + } + + this.l1Queue.add(async () => { + await this.runL1(sessionKey); + }).catch((err) => { + this.logger?.error( + `${TAG} [${sessionKey}] L1 task failed: ${err instanceof Error ? err.stack ?? err.message : String(err)}`, + ); + }).finally(() => { + timers.l1Queued = false; + }); + } + + /** + * L1 runner: Takes all buffered messages for a session and passes them + * to the L1Runner for batch processing (e.g. appendEvent, local extraction). + * + * After L1 completes successfully: + * - conversation_count and message buffer are reset + * - L2 timer is advanced (downward-only) to allow remote record generation + * + * If L1 fails, conversation_count and buffer are preserved for retry + * on next idle timeout or threshold trigger. + */ + private async runL1(sessionKey: string): Promise { + const state = this.sessionStates.get(sessionKey); + if (!state) return; + + // Drain the message buffer (take ownership, clear the shared ref) + const buffer = this.messageBuffers.get(sessionKey) ?? []; + this.messageBuffers.set(sessionKey, []); + + if (buffer.length === 0 && state.conversation_count === 0) { + this.logger?.debug?.(`${TAG} [${sessionKey}] L1 skipped: no messages and no pending conversations`); + return; + } + + this.logger?.debug?.( + `${TAG} [${sessionKey}] L1 running: messages=${buffer.length}, conversation_count=${state.conversation_count}`, + ); + + if (!this.l1Runner) { + this.logger?.warn(`${TAG} [${sessionKey}] No L1 runner set, skipping`); + state.l2_pending_l1_count = state.conversation_count; + state.conversation_count = 0; + this.advanceWarmupThreshold(state); + await this.persistStates(); + this.advanceL2Timer(sessionKey); + return; + } + + try { + await this.l1Runner({ + sessionKey, + msg: buffer, + bg_msg: [], // reserved for future use + }); + + this.logger?.debug?.( + `${TAG} [${sessionKey}] L1 complete: processed ${buffer.length} messages`, + ); + } catch (err) { + this.logger?.error( + `${TAG} [${sessionKey}] L1 runner failed: ${err instanceof Error ? err.stack ?? err.message : String(err)}`, + ); + // On failure: put messages back into the buffer for retry + const currentBuffer = this.messageBuffers.get(sessionKey) ?? []; + this.messageBuffers.set(sessionKey, [...buffer, ...currentBuffer]); + this.logger?.debug?.( + `${TAG} [${sessionKey}] L1 failure: restored ${buffer.length} messages to buffer (total=${buffer.length + currentBuffer.length})`, + ); + + // Re-arm L1 idle timer for automatic retry (with max retry limit) + const timers = this.getOrCreateTimers(sessionKey); + timers.l1RetryCount += 1; + if (timers.l1RetryCount <= this.L1_MAX_RETRIES) { + timers.l1Idle.schedule(this.L1_RETRY_DELAY_MS, () => this.onL1IdleTimeout(sessionKey)); + this.logger?.debug?.( + `${TAG} [${sessionKey}] L1 retry scheduled in ${this.L1_RETRY_DELAY_MS / 1000}s ` + + `(attempt ${timers.l1RetryCount}/${this.L1_MAX_RETRIES})`, + ); + } else { + this.logger?.warn( + `${TAG} [${sessionKey}] L1 max retries reached (${this.L1_MAX_RETRIES}), ` + + `giving up auto-retry. ${buffer.length + currentBuffer.length} messages remain buffered. ` + + `Will resume on next user conversation.`, + ); + } + + return; // don't advance state or trigger L2 + } + + // Success: reset retry count and advance state + const timers = this.getOrCreateTimers(sessionKey); + timers.l1RetryCount = 0; + state.l2_pending_l1_count = state.conversation_count; + state.conversation_count = 0; + this.advanceWarmupThreshold(state); + await this.persistStates(); + + // Advance the L2 timer (downward-only) to fire after delay, respecting minInterval + this.advanceL2Timer(sessionKey); + } + + // ============================ + // Internal: L2 timer management (downward-only) + // ============================ + + /** + * Advance the per-session L2 timer after an L1 event (new memory generated). + * + * Computes the desired fire time as: + * T_desired = max(now + l2DelayAfterL1, lastL2Time + l2MinInterval) + * + * The timer is only moved if T_desired is earlier than the current schedule + * (downward-only semantics). If no timer is pending, it's set unconditionally. + */ + private advanceL2Timer(sessionKey: string): void { + if (this.destroyed) return; + + const timers = this.getOrCreateTimers(sessionKey); + const now = Date.now(); + + // Compute the floor: lastL2 + minInterval (rate-limit protection) + const lastL2 = this.l2LastRunTime.get(sessionKey) ?? 0; + const minIntervalFloor = lastL2 > 0 ? lastL2 + this.l2MinIntervalMs : 0; + + // Desired fire time: delay after L1, but no earlier than minInterval floor + const desiredTime = Math.max(now + this.l2DelayAfterL1Ms, minIntervalFloor); + + const advanced = timers.l2Schedule.tryAdvanceTo(desiredTime, () => this.onL2TimerFired(sessionKey, "delay-after-l1")); + + if (advanced) { + const delaySec = Math.round((desiredTime - now) / 1000); + this.logger?.debug?.( + `${TAG} [${sessionKey}] L2 timer advanced: firing in ${delaySec}s` + + (timers.l2Schedule.scheduledTime > 0 + ? ` (was ${Math.round((timers.l2Schedule.scheduledTime - now) / 1000)}s)` + : " (newly armed)"), + ); + } else { + this.logger?.debug?.( + `${TAG} [${sessionKey}] L2 timer not advanced: current schedule is already earlier`, + ); + } + } + + /** + * Arm the L2 timer for the maxInterval guarantee after L2 completes. + * Sets T = now + l2MaxInterval (unconditional, replaces any pending timer). + */ + private armL2MaxInterval(sessionKey: string): void { + if (this.destroyed) return; + + const timers = this.getOrCreateTimers(sessionKey); + const fireAt = Date.now() + this.l2MaxIntervalMs; + timers.l2Schedule.scheduleAt(fireAt, () => this.onL2TimerFired(sessionKey, "max-interval")); + + this.logger?.debug?.( + `${TAG} [${sessionKey}] L2 maxInterval timer armed: ${Math.round(this.l2MaxIntervalMs / 1000)}s`, + ); + } + + /** + * Called when a per-session L2 timer fires. + * + * Checks session activity: if the session is cold (inactive > activeWindow), + * the timer is NOT re-armed — it will be revived by the next L1 event. + * Otherwise, enqueues L2. + * + * The `source` parameter distinguishes the trigger origin: + * - "delay-after-l1": fired shortly after L1 completed — skip cold check + * because L1 completion itself proves recent activity. + * - "max-interval": periodic timer — apply cold check normally. + */ + private onL2TimerFired(sessionKey: string, source: "delay-after-l1" | "max-interval"): void { + const state = this.sessionStates.get(sessionKey); + if (!state) return; + + const now = Date.now(); + + // Cold session check: only applies to periodic (maxInterval) triggers. + // Delay-after-L1 triggers are exempt because L1 just completed, proving + // the session was recently active. + if (source === "max-interval" && now - state.last_active_time >= this.sessionActiveWindowMs) { + this.logger?.debug?.( + `${TAG} [${sessionKey}] L2 timer fired but session is cold ` + + `(inactive ${Math.round((now - state.last_active_time) / 3600_000)}h), timer stopped. ` + + `Will re-arm on next L1 event.`, + ); + return; // timer not re-armed — advanceL2Timer() in runL1 will revive it + } + + this.enqueueL2(sessionKey, `timer:${source}`); + } + + // ============================ + // Internal: L2 queue + // ============================ + + private enqueueL2(sessionKey: string, trigger: string): void { + const timers = this.getOrCreateTimers(sessionKey); + + // Cancel any pending L2 timer (we're about to run L2) + timers.l2Schedule.cancel(); + + // Conflict detection: warn if L2 is already queued + if (timers.l2Queued) { + this.logger?.warn( + `${TAG} [${sessionKey}] L2 enqueue conflict on queue "${this.l2Queue.name}": ` + + `task already queued/running (trigger=${trigger}), skipping`, + ); + return; + } + + timers.l2Queued = true; + this.logger?.debug?.(`${TAG} [${sessionKey}] Enqueuing L2 (trigger=${trigger}, queue=${this.l2Queue.name})`); + + this.l2Queue.add(async () => { + await this.runL2(sessionKey); + }).catch((err) => { + this.logger?.error( + `${TAG} [${sessionKey}] L2 task failed: ${err instanceof Error ? err.stack ?? err.message : String(err)}`, + ); + }).finally(() => { + timers.l2Queued = false; + }); + } + + private async runL2(sessionKey: string): Promise { + const state = this.sessionStates.get(sessionKey); + if (!state) return; + + if (!this.l2Runner) { + this.logger?.warn(`${TAG} [${sessionKey}] No L2 runner set, skipping`); + return; + } + + this.logger?.debug?.( + `${TAG} [${sessionKey}] L2 running: l2_pending_l1_count=${state.l2_pending_l1_count}`, + ); + + const cursor = state.last_extraction_updated_time || undefined; + + let result: L2RunnerResult | void; + try { + result = await this.l2Runner(sessionKey, cursor); + } catch (err) { + this.logger?.error( + `${TAG} [${sessionKey}] L2 runner failed: ${err instanceof Error ? err.stack ?? err.message : String(err)}`, + ); + // Even on failure, arm maxInterval so we retry eventually + this.armL2MaxInterval(sessionKey); + return; + } + + // After L2: update state + const now = Date.now(); + state.l2_pending_l1_count = 0; + state.last_extraction_time = new Date().toISOString(); + state.l2_last_extraction_time = new Date().toISOString(); + this.l2LastRunTime.set(sessionKey, now); + + // Advance cursor using the record timestamp returned by the runner + if (result?.latestCursor) { + state.last_extraction_updated_time = result.latestCursor; + } + + await this.persistStates(); + + this.logger?.debug?.(`${TAG} [${sessionKey}] L2 complete`); + + // Arm the maxInterval timer for the next cycle + this.armL2MaxInterval(sessionKey); + + // Trigger L3 + this.triggerL3(); + } + + // ============================ + // Internal: L3 queue (global, dedup) + // ============================ + + private triggerL3(): void { + if (this.destroyed) return; + + if (this.l3Running) { + // L3 is in progress — mark pending so it runs again after current finishes + this.l3Pending = true; + this.logger?.debug?.(`${TAG} L3 already running, marking pending`); + return; + } + + this.logger?.debug?.(`${TAG} Triggering L3`); + this.enqueueL3(); + } + + private enqueueL3(): void { + this.l3Running = true; + this.l3Pending = false; + + this.logger?.debug?.(`${TAG} Enqueuing L3 (queue=${this.l3Queue.name})`); + + this.l3Queue.add(async () => { + await this.runL3(); + }).catch((err) => { + this.logger?.error( + `${TAG} L3 task failed: ${err instanceof Error ? err.stack ?? err.message : String(err)}`, + ); + }).finally(() => { + this.l3Running = false; + + // If new L2 completions happened while L3 was running, run again + if (this.l3Pending && !this.destroyed) { + this.logger?.debug?.(`${TAG} L3 has pending work, re-running`); + this.enqueueL3(); + } + }); + } + + private async runL3(): Promise { + if (!this.l3Runner) { + this.logger?.warn(`${TAG} No L3 runner set, skipping`); + return; + } + + this.logger?.debug?.(`${TAG} L3 running`); + try { + await this.l3Runner(); + this.logger?.debug?.(`${TAG} L3 complete`); + } catch (err) { + this.logger?.error( + `${TAG} L3 runner failed: ${err instanceof Error ? err.stack ?? err.message : String(err)}`, + ); + } + } + + // ============================ + // Internal: state management + // ============================ + + private getOrCreateState(sessionKey: string): PipelineSessionState { + let state = this.sessionStates.get(sessionKey); + if (!state) { + state = { + conversation_count: 0, + last_extraction_time: "", + last_extraction_updated_time: "", + last_active_time: Date.now(), + l2_pending_l1_count: 0, + warmup_threshold: this.enableWarmup ? 1 : 0, + l2_last_extraction_time: "", + }; + this.sessionStates.set(sessionKey, state); + this.logger?.debug?.(`${TAG} [${sessionKey}] Created new session state`); + } + return state; + } + + private getOrCreateTimers(sessionKey: string): SessionTimerState { + let timers = this.sessionTimers.get(sessionKey); + if (!timers) { + const isDestroyed = () => this.destroyed; + timers = { + l1Idle: new ManagedTimer(`L1-idle:${sessionKey}`, isDestroyed), + l2Schedule: new ManagedTimer(`L2-schedule:${sessionKey}`, isDestroyed), + l1Queued: false, + l2Queued: false, + l1RetryCount: 0, + }; + this.sessionTimers.set(sessionKey, timers); + } + return timers; + } + + private async persistStates(): Promise { + if (!this.persister) return; + + // PipelineSessionState only contains pipeline-owned fields, so we can + // safely persist the entire object without risk of overwriting runner state. + const obj: Record = {}; + for (const [k, v] of this.sessionStates) { + obj[k] = { ...v }; + } + try { + this.logger?.debug?.(`Persisting states: ${JSON.stringify(obj)}`); + await this.persister(obj); + } catch (err) { + this.logger?.error( + `${TAG} Failed to persist states: ${err instanceof Error ? err.message : String(err)}`, + ); + } + } + + /** + * Evict cold sessions from in-memory maps to prevent unbounded growth. + * + * A session is eligible for GC when: + * 1. Inactive for > sessionActiveWindowMs * SESSION_GC_INACTIVE_MULTIPLIER + * 2. No queued/running L1 or L2 tasks + * 3. No buffered messages pending processing + * + * Evicted sessions can be fully restored from checkpoint on next + * `notifyConversation()` (state) or `start()` (recovery). + */ + private gcStaleSessions(): void { + const now = Date.now(); + const maxInactiveMs = this.sessionActiveWindowMs * this.SESSION_GC_INACTIVE_MULTIPLIER; + let evictedCount = 0; + + for (const [sessionKey, state] of this.sessionStates) { + if (now - state.last_active_time < maxInactiveMs) continue; + + // Safety: don't evict sessions with active work + const timers = this.sessionTimers.get(sessionKey); + if (timers?.l1Queued || timers?.l2Queued) continue; + + const buffer = this.messageBuffers.get(sessionKey); + if (buffer && buffer.length > 0) continue; + + // Evict: cancel any pending timers, then remove from all maps + if (timers) { + timers.l1Idle.cancel(); + timers.l2Schedule.cancel(); + } + this.sessionStates.delete(sessionKey); + this.sessionTimers.delete(sessionKey); + this.messageBuffers.delete(sessionKey); + this.l2LastRunTime.delete(sessionKey); + evictedCount++; + } + + if (evictedCount > 0) { + this.logger?.debug?.( + `${TAG} Session GC: evicted ${evictedCount} cold session(s), ` + + `${this.sessionStates.size} remaining`, + ); + } + } + + /** + * Recovery: re-enqueue sessions that have pending work from before restart. + * + * On restart, message buffers are empty (in-memory only). Sessions with + * non-zero conversation_count had messages that were either: + * 1. Already processed by L1 (l2_pending_l1_count > 0) → arm L2 timer + * 2. Never reached L1 (conversation_count > 0, messages lost) → arm L2 + * as best-effort recovery + * + * We arm L2 timers (with delay) rather than enqueuing immediately, + * because the pipeline may be starting during management commands. + */ + private recoverPendingSessions(): void { + for (const [sessionKey, state] of this.sessionStates) { + if (state.conversation_count === 0 && state.l2_pending_l1_count === 0) continue; + + this.logger?.debug?.( + `${TAG} [${sessionKey}] Recovery: conversation_count=${state.conversation_count}, ` + + `l2_pending_l1_count=${state.l2_pending_l1_count}, arming L2 timer`, + ); + + // Reset conversation_count since we can't recover the messages + state.l2_pending_l1_count = Math.max(state.l2_pending_l1_count, state.conversation_count); + state.conversation_count = 0; + + // Arm L2 timer with delay (gives the system time to fully start) + this.advanceL2Timer(sessionKey); + } + } + + // ============================ + // Public accessors (for testing / status) + // ============================ + + /** Get the pipeline session state for a session (read-only copy). */ + getSessionState(sessionKey: string): PipelineSessionState | undefined { + const state = this.sessionStates.get(sessionKey); + return state ? { ...state } : undefined; + } + + /** Get the buffered message count for a session. */ + getBufferedMessageCount(sessionKey: string): number { + return this.messageBuffers.get(sessionKey)?.length ?? 0; + } + + /** Get all session keys being tracked. */ + getSessionKeys(): string[] { + return Array.from(this.sessionStates.keys()); + } + + /** Whether the pipeline has been destroyed. */ + get isDestroyed(): boolean { + return this.destroyed; + } + + /** Queue sizes for monitoring. */ + getQueueSizes(): { l1: number; l2: number; l3: number } { + return { + l1: this.l1Queue.size, + l2: this.l2Queue.size, + l3: this.l3Queue.size, + }; + } +} diff --git a/src/utils/sanitize.ts b/src/utils/sanitize.ts new file mode 100644 index 0000000..05097bc --- /dev/null +++ b/src/utils/sanitize.ts @@ -0,0 +1,398 @@ +/** + * Text sanitization for memory pipeline (capture & recall). + * Removes injected tags, gateway metadata, media noise, etc. + */ + +/** + * Clean text for the memory pipeline: remove injected tags, metadata, + * timestamps, media markers and base64 image data. + * + * Used by both capture (L0 recording) and recall (query cleaning) paths. + */ +export function sanitizeText(text: string): string { + let cleaned = text; + + // Remove injected memory context tags (prevent feedback loops) + cleaned = cleaned.replace(/[\s\S]*?<\/relevant-memories>/g, ""); + cleaned = cleaned.replace(/[\s\S]*?<\/user-persona>/g, ""); + cleaned = cleaned.replace(/[\s\S]*?<\/relevant-scenes>/g, ""); + cleaned = cleaned.replace(/[\s\S]*?<\/scene-navigation>/g, ""); + + // Remove framework-injected inbound metadata blocks (from inbound-meta.ts buildInboundUserContextPrefix). + // These are "label:\n```json\n...\n```" blocks that the framework prepends to user messages. + // Pattern matches all known block labels: + // - Conversation info (untrusted metadata): + // - Sender (untrusted metadata): + // - Thread starter (untrusted, for context): + // - Replied message (untrusted, for context): + // - Forwarded message context (untrusted metadata): + // - Chat history since last reply (untrusted, for context): + cleaned = cleaned.replace( + /(?:Conversation info|Sender|Thread starter|Replied message|Forwarded message context|Chat history since last reply)\s*\(untrusted[\s\S]*?\):\s*```json\s*[\s\S]*?```/g, + "", + ); + + // Remove conversation metadata JSON blocks (legacy pattern) + cleaned = cleaned.replace(/```json\s*\{[\s\S]*?"session[\s\S]*?\}\s*```/g, ""); + + // Remove framework reply directive tags: [[reply_to_current]], [[reply_to_xxx]], etc. + cleaned = cleaned.replace(/\[\[reply_to[^\]]*\]\]\s*/g, ""); + + // Remove line-leading timestamps, e.g. "[Tue 2026-03-24 03:48 UTC]" + // or "[Tue 2026-03-24 20:21 GMT+8]", "[Thu 2026-03-24 01:51 GMT+5:30]" + // Matches brackets containing word chars, digits, hyphens, colons, plus signs, + // and spaces — the '+' is needed for timezone offsets like GMT+8, GMT+5:30. + cleaned = cleaned.replace(/^\[[\w\d\-:+ ]+\]\s*/gm, ""); + + // Remove gateway media-attachment markers: + // [media attached: /path/to/file.png (image/png) | /path/to/file.png] + cleaned = cleaned.replace(/\[media attached:[^\]]*\]\s*/g, ""); + + // Remove gateway image-reply instructions injected after media attachments. + // Starts with "To send an image back" and ends before the next real content. + cleaned = cleaned.replace( + /To send an image back,[\s\S]*?(?:Keep caption in the text body\.)\s*/g, + "", + ); + + // Remove "System: [timestamp] Exec completed ..." blocks appended by the framework. + cleaned = cleaned.replace(/^System:\s*\[[\s\S]*?$/gm, ""); + + // Remove inline base64 image data URIs (e.g. data:image/png;base64,iVBOR...) + // Replace with empty string (not a placeholder) so that pure-image messages + // become empty after sanitization and are naturally filtered by length checks. + cleaned = cleaned.replace(/data:image\/[a-z+]+;base64,[A-Za-z0-9+/=]+/gi, ""); + + // Remove null chars + compress whitespace + cleaned = cleaned.replace(/\0/g, "").replace(/\n{3,}/g, "\n\n").trim(); + + return cleaned; +} + +/** + * Strip fenced code blocks from assistant replies before L0 capture. + * + * AI responses often contain large code snippets (```...```) that dilute + * the semantic signal for embedding and memory extraction. This function + * removes only the code block content while preserving surrounding + * natural-language explanations. + * + * Only applied to `role=assistant` messages in the L0 capture path — + * user messages and recall queries are NOT affected. + */ +export function stripCodeBlocks(text: string): string { + return text.replace(/```[^\n]*\n[\s\S]*?```/g, "").replace(/\n{3,}/g, "\n\n").trim(); +} + +// ============================ +// L0 / L1 Capture & Extraction Filters +// ============================ + +/** + * L0 capture filter — intentionally **permissive**. + * + * L0 is the raw conversation archive. We want to preserve as much user input + * as possible so that downstream stages (L1 extraction, search, analytics) + * have the full picture. Only messages that are *structurally* useless are + * dropped here: + * - Empty / whitespace-only text + * - Framework-internal noise (bootstrap, session reset, NO_REPLY, …) + * - Slash commands (/new, /reset, …) + * + * Content-quality filters (length, symbols, prompt injection) are deferred + * to {@link shouldExtractL1}. + */ +export function shouldCaptureL0(text: string): boolean { + if (!text || !text.trim()) return false; + + // Filter framework-internal / bootstrap noise messages + if (isFrameworkNoise(text)) return false; + + // Slash commands are framework directives, not user content + if (text.startsWith("/")) return false; + + return true; +} + +/** + * L1 extraction filter — **strict** quality gate. + * + * Applied when L0 messages are fed into the LLM extraction pipeline. + * Filters out content that is too short, too long, purely symbolic, + * or looks like a prompt-injection attack — none of which should + * become structured memories. + * + * This function is a superset of {@link shouldCaptureL0}: anything + * rejected by L0 is also rejected here, plus additional quality checks. + */ +export function shouldExtractL1(text: string): boolean { + // First apply the same structural filters as L0 + if (!shouldCaptureL0(text)) return false; + + // ── Length filters ── + // const isCJK = /[\u4e00-\u9fff\u3040-\u30ff\uac00-\ud7af]/.test(text); + // if (isCJK && text.length < 2) return false; + // if (!isCJK && text.length < 2) return false; + // if (text.length > 5000) return false; + + // ── Content-quality filters ── + // Match strings composed entirely of non-word, non-space, non-CJK characters (1–5 chars). + if (/^[^\w\s\u4e00-\u9fff\u3040-\u30ff\uac00-\ud7af]{1,5}$/.test(text)) return false; + if (/^[?？]+$/.test(text)) return false; + + // ── Security filters ── + // Reject prompt-injection payloads — prevent malicious content from being + // persisted into structured memory and re-injected on future recalls. + // if (looksLikePromptInjection(text)) return false; + + return true; +} + +/** + * @deprecated Use {@link shouldExtractL1} (strict) or {@link shouldCaptureL0} (permissive) instead. + * + * Kept as an alias of `shouldExtractL1` for backward compatibility. + */ +export const shouldCapture = shouldExtractL1; + +// ============================ +// Prompt Injection Detection +// ============================ + +/** + * Known prompt-injection / jailbreak patterns. + * + * Covers: + * 1. Instruction override — "ignore all previous instructions", etc. + * 2. Role hijack — "you are now DAN", "act as root", etc. + * 3. System/developer boundary probing — "system prompt", "developer message" + * 4. XML/tag injection — opening tags that match our context boundaries + * 5. Tool/command invocation tricks — "run command X", "execute tool Y" + * 6. Multi-language variants — Chinese prompt-injection patterns + */ +const PROMPT_INJECTION_PATTERNS: RegExp[] = [ + // ── Instruction override ── + /ignore\b.{0,30}\b(instructions|rules|guidelines)/i, + /disregard\b.{0,30}\b(instructions|rules|guidelines)/i, + /forget\b.{0,30}\b(instructions|rules|context)/i, + /override\b.{0,30}\b(instructions|rules|guidelines|safety)/i, + + // ── Role hijack ── + /you are now (?!going|about|ready)/i, // "you are now DAN" but not "you are now going to..." + /act as (?:if you are |if you were )?(?:a |an )?(?:root|admin|unrestricted|unfiltered|jailbroken)/i, + /enter (?:DAN|jailbreak|god|sudo|developer|dev|debug|unrestricted|unfiltered) mode/i, + /switch to (?:DAN|jailbreak|god|sudo|developer|dev|debug|unrestricted|unfiltered) mode/i, + + // ── System boundary probing ── + /(?:show|reveal|print|output|display|repeat|leak|dump|give)\b.{0,20}\bsystem prompt/i, + /reveal (?:your |the )?(system|hidden|secret|internal) (?:prompt|instructions|rules)/i, + /what (?:are|is) your (?:system|hidden|original|initial) (?:prompt|instructions|rules)/i, + + // ── XML/tag injection (our context boundaries) ── + /<\s*(system|assistant|developer|tool|function|relevant-memories)\b/i, + + // ── Tool/command invocation tricks ── + /\b(run|execute|call|invoke)\b.{0,40}\b(tool|command|function|shell)\b/i, + + // ── Chinese variants ── + /忽略(?:所有|之前|以上|先前)?(?:的)?(?:指令|规则|指示|说明)/, + /无视(?:所有|之前|以上)?(?:的)?(?:指令|规则|限制)/, + /(?:显示|输出|告诉我|给我看)(?:你的)?(?:系统|初始|隐藏)?(?:提示词|指令|规则|prompt)/, + /你(?:现在|从现在开始)是/, // "你现在是 DAN" +]; + +/** + * Detect likely prompt-injection / jailbreak attempts. + * + * Normalises whitespace before matching to defeat trivial obfuscation + * (e.g. extra spaces / newlines between keywords). + */ +export function looksLikePromptInjection(text: string): boolean { + const normalized = text.replace(/\s+/g, " ").trim(); + if (!normalized) return false; + return PROMPT_INJECTION_PATTERNS.some((pattern) => pattern.test(normalized)); +} + +/** + * Detect framework-injected noise messages that should never be captured. + * + * These include: + * - "(session bootstrap)" — synthetic user turn for Google turn-order compliance + * - Session startup instructions from /new or /reset + * - "✅ New session started" — AI's ack of session startup (no user-meaningful content) + * - Pre-compaction memory flush prompts (system-to-agent instructions, not user content) + * - AI's NO_REPLY ack of memory flush (no user-meaningful content) + */ +function isFrameworkNoise(text: string): boolean { + const t = text.trim(); + + // Google turn-order bootstrap placeholder + if (t === "(session bootstrap)") return true; + + // Framework session-reset instruction (starts with "A new session was started via /new or /reset") + if (t.startsWith("A new session was started via")) return true; + + // AI's pure ack of session startup: "✅ New session started · model: ..." + if (/^✅\s*New session started/.test(t)) return true; + + // Pre-compaction memory flush prompt injected by the framework as a synthetic + // user turn. This is an internal system-to-agent instruction, NOT real user + // content. Capturing it would pollute L0/L1 memories with framework directives. + if (t.startsWith("Pre-compaction memory flush")) return true; + + // AI's NO_REPLY response to memory flush (or other silent-reply scenarios). + // A bare "NO_REPLY" (with optional whitespace) carries no user-meaningful content. + if (/^NO_REPLY\s*$/.test(t)) return true; + + return false; +} + +/** + * Pick up to `max` recent unique texts. + */ +export function pickRecentUnique(texts: string[], max: number): string[] { + const seen = new Set(); + const result: string[] = []; + for (let i = texts.length - 1; i >= 0 && result.length < max; i--) { + const t = texts[i]!; + if (!seen.has(t)) { + seen.add(t); + result.push(t); + } + } + return result.reverse(); +} + +// ============================ +// LLM Safety Utilities +// ============================ + +/** + * Escape XML-like tags in text to prevent tag injection attacks. + * + * When memory content or persona text is injected into XML-delimited sections + * (e.g. `...`), a malicious user could craft content + * containing `` to break out of the section boundary. + * + * This function escapes `<` and `>` in known dangerous patterns (closing tags + * that match our injection boundaries) so the content cannot prematurely close + * the XML section. + */ +export function escapeXmlTags(text: string): string { + // Escape closing tags that match our injection section boundaries + return text.replace( + /<\/?(?:user-persona|relevant-memories|scene-navigation|relevant-scenes|memory-tools-guide|system|assistant)>/gi, + (match) => match.replace(//g, ">"), + ); +} + +// ============================ +// JSON Sanitization for LLM Output +// ============================ + +/** + * Sanitize a raw JSON string from LLM output so that `JSON.parse` won't throw + * "Bad control character in string literal". + * + * Per RFC 8259 §7, U+0000–U+001F MUST be escaped inside JSON string literals. + * LLMs sometimes produce unescaped control characters (raw newlines, tabs, etc.) + * inside string values. + * + * Strategy (two-phase): + * 1. **Precise pass** — walk through JSON string literals (delimited by `"`) + * and escape any unescaped U+0000–U+001F inside them to `\uXXXX` form, + * while leaving structural whitespace (between values) untouched. + * 2. **Fallback** — if the precise pass still fails `JSON.parse`, fall back to + * a simple global strip of rare control chars (\x00–\x08, \x0b, \x0c, + * \x0e–\x1f) which are almost never meaningful in natural-language content. + */ +export function sanitizeJsonForParse(raw: string): string { + // Phase 1: Escape control characters inside JSON string literals. + // We walk the string character-by-character to properly handle escape sequences. + const escaped = escapeControlCharsInJsonStrings(raw); + try { + JSON.parse(escaped); + return escaped; + } catch { + // Phase 1 didn't fully fix it — fall through to phase 2 + } + + // Phase 2: Brute-force strip of rare control chars that have no textual meaning. + // Preserves \t (\x09), \n (\x0a), \r (\x0d) which are common structural whitespace. + // NOTE: We strip from `escaped` (Phase 1 result) rather than `raw`, so that any + // control-character escaping Phase 1 performed is preserved even when the JSON has + // other issues (e.g. trailing commas) that cause the Phase 1 parse to fail. + const stripped = escaped.replace(/[\x00-\x08\x0b\x0c\x0e-\x1f]/g, ""); + return stripped; +} + +/** + * Walk through a JSON text and escape U+0000–U+001F control characters that + * appear *inside* JSON string literals (between unescaped `"` delimiters). + * + * Characters that already have short escape sequences (\n, \r, \t, \b, \f) + * are mapped to those; others become \uXXXX. + * + * Structural whitespace outside string literals is left untouched. + */ +function escapeControlCharsInJsonStrings(text: string): string { + const SHORT_ESCAPES: Record = { + 0x08: "\\b", // backspace + 0x09: "\\t", // tab + 0x0a: "\\n", // line feed + 0x0c: "\\f", // form feed + 0x0d: "\\r", // carriage return + }; + + const out: string[] = []; + let inString = false; + let i = 0; + + while (i < text.length) { + const ch = text[i]!; + const code = ch.charCodeAt(0); + + if (inString) { + if (ch === "\\" && i + 1 < text.length) { + // Already-escaped sequence — copy both characters verbatim + out.push(ch, text[i + 1]!); + i += 2; + continue; + } + if (ch === '"') { + // End of string literal + out.push(ch); + inString = false; + i++; + continue; + } + if (code <= 0x1f) { + // Unescaped control character inside string — escape it + const short = SHORT_ESCAPES[code]; + if (short) { + out.push(short); + } else { + out.push("\\u" + code.toString(16).padStart(4, "0")); + } + i++; + continue; + } + // Normal character inside string + out.push(ch); + i++; + } else { + // Outside string literal + if (ch === '"') { + out.push(ch); + inString = true; + i++; + continue; + } + // Structural character (including whitespace) — pass through + out.push(ch); + i++; + } + } + + return out.join(""); +} diff --git a/src/utils/serial-queue.ts b/src/utils/serial-queue.ts new file mode 100644 index 0000000..dca2b94 --- /dev/null +++ b/src/utils/serial-queue.ts @@ -0,0 +1,120 @@ +/** + * SerialQueue: a lightweight task queue with concurrency=1. + * + * Equivalent to `new PQueue({ concurrency: 1 })` but with zero external + * dependencies. Supports: + * - Serial execution (FIFO) + * - `add(fn)` to enqueue a task (returns the task's result promise) + * - `onIdle()` to wait until all queued tasks have completed + * - `pause()` / `start()` to suspend/resume execution + * - `size` to check pending task count + * - Optional debug logger for enqueue/dequeue/complete diagnostics + */ + +type Task = () => Promise; + +interface QueueEntry { + task: Task; + resolve: (value: unknown) => void; + reject: (reason: unknown) => void; +} + +export class SerialQueue { + /** Human-readable name for logging / diagnostics. */ + public readonly name: string; + + private queue: QueueEntry[] = []; + private running = false; + private paused = false; + private idleResolvers: Array<() => void> = []; + + /** Optional debug logger — receives diagnostic messages for enqueue/dequeue/complete. */ + private debugFn?: (msg: string) => void; + + constructor(name = "unnamed") { + this.name = name; + } + + /** Set a debug logger for queue diagnostics. */ + setDebugLogger(fn: (msg: string) => void): void { + this.debugFn = fn; + } + + /** Number of tasks waiting to be executed. */ + get size(): number { + return this.queue.length; + } + + /** Whether a task is currently executing. */ + get pending(): boolean { + return this.running; + } + + /** Add a task to the queue. Returns the task's result promise. */ + add(task: Task): Promise { + return new Promise((resolve, reject) => { + this.queue.push({ + task: task as Task, + resolve: resolve as (value: unknown) => void, + reject, + }); + this.debugFn?.(`[queue:${this.name}] enqueued, pending=${this.queue.length}, running=${this.running}`); + this.drain(); + }); + } + + /** Pause the queue. Currently running task will finish, but no new tasks start. */ + pause(): void { + this.paused = true; + } + + /** Resume the queue after pause(). */ + start(): void { + this.paused = false; + this.drain(); + } + + /** Returns a promise that resolves when all queued tasks have completed. */ + onIdle(): Promise { + if (this.queue.length === 0 && !this.running) { + return Promise.resolve(); + } + return new Promise((resolve) => { + this.idleResolvers.push(resolve); + }); + } + + /** Clear all pending (not yet started) tasks. */ + clear(): void { + for (const entry of this.queue) { + entry.reject(new Error("Queue cleared")); + } + this.queue = []; + } + + private drain(): void { + if (this.running || this.paused || this.queue.length === 0) return; + + const entry = this.queue.shift()!; + this.running = true; + + this.debugFn?.(`[queue:${this.name}] dequeued, starting execution (remaining=${this.queue.length})`); + + entry + .task() + .then((result) => entry.resolve(result)) + .catch((err) => entry.reject(err)) + .finally(() => { + this.running = false; + this.debugFn?.(`[queue:${this.name}] task completed (remaining=${this.queue.length})`); + if (this.queue.length === 0) { + // Notify idle waiters + const resolvers = this.idleResolvers; + this.idleResolvers = []; + for (const resolve of resolvers) resolve(); + } else { + this.drain(); + } + }); + } +} diff --git a/src/utils/session-filter.ts b/src/utils/session-filter.ts new file mode 100644 index 0000000..cf4fe2f --- /dev/null +++ b/src/utils/session-filter.ts @@ -0,0 +1,108 @@ +/** + * Session filtering for memory-tdai. + * + * Decides whether a session should be ignored by the memory plugin + * (capture, recall, pipeline scheduling). All skip rules are compiled + * into a flat list of matchers at construction time — zero per-call overhead. + */ + +// ============================ +// Types +// ============================ + +export interface AgentHookContext { + sessionKey?: string; + sessionId?: string; + trigger?: string; +} + +type SessionKeyMatcher = (sessionKey: string) => boolean; + +// ============================ +// Non-interactive trigger detection +// ============================ + +const SKIP_TRIGGERS = new Set(["cron", "heartbeat", "automation", "schedule"]); + +/** + * Returns true when the hook was fired by a non-interactive trigger + * (heartbeat, cron job, automation, etc.) — these produce no meaningful + * user conversation and should not be captured or counted. + */ +export function isNonInteractiveTrigger(trigger?: string, sessionKey?: string): boolean { + if (trigger && SKIP_TRIGGERS.has(trigger.toLowerCase())) return true; + if (sessionKey) { + if (/:cron:/i.test(sessionKey) || /:heartbeat:/i.test(sessionKey)) return true; + } + return false; +} + +// ============================ +// Built-in skip rules (always active) +// ============================ + +/** + * Hard-coded matchers that identify internal / non-user sessions. + * These are always applied regardless of user configuration. + */ +const BUILTIN_MATCHERS: SessionKeyMatcher[] = [ + // Scene extraction runner sessions + (key) => key.includes(":memory-scene-extract-"), + // OpenClaw subagent sessions + (key) => key.includes(":subagent:"), + // Temporary / internal utility sessions (e.g. temp:slug-generator) + (key) => key.startsWith("temp:"), +]; + +// ============================ +// Glob → matcher compiler +// ============================ + +/** + * Turn a simple glob pattern (only `*` supported) into a matcher + * that tests the full sessionKey. + * + * Since sessionKeys look like `agent::...`, we match the + * glob against the whole key so users can write patterns like + * `bench-judge-*` (matched anywhere) or more specific ones. + */ +function globToMatcher(pattern: string): SessionKeyMatcher { + const escaped = pattern.replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*"); + const re = new RegExp(escaped); + return (key) => re.test(key); +} + +// ============================ +// SessionFilter +// ============================ + +/** + * Unified filter: construct once at plugin startup, then call + * `shouldSkip(sessionKey)` or `shouldSkipCtx(ctx)` at each gate. + */ +export class SessionFilter { + private readonly matchers: SessionKeyMatcher[]; + + constructor(excludeAgents: string[] = []) { + // Merge built-in rules + user-configured exclude patterns into one flat list + const userMatchers = excludeAgents + .map((p) => p.trim()) + .filter((p) => p.length > 0) + .map(globToMatcher); + + this.matchers = [...BUILTIN_MATCHERS, ...userMatchers]; + } + + /** Should this sessionKey be skipped? */ + shouldSkip(sessionKey: string): boolean { + return this.matchers.some((m) => m(sessionKey)); + } + + /** Should this hook context be skipped? */ + shouldSkipCtx(ctx: AgentHookContext): boolean { + if (!ctx.sessionKey) return true; + if (ctx.sessionId?.startsWith("memory-")) return true; + if (isNonInteractiveTrigger(ctx.trigger, ctx.sessionKey)) return true; + return this.shouldSkip(ctx.sessionKey); + } +} diff --git a/src/utils/text-utils.ts b/src/utils/text-utils.ts new file mode 100644 index 0000000..306c139 --- /dev/null +++ b/src/utils/text-utils.ts @@ -0,0 +1,31 @@ +/** + * Shared text utility functions for the memory-tdai plugin. + */ + +/** + * Extract meaningful words from text (supports CJK and Latin). + * + * Used by both auto-recall (keyword search) and l1-dedup (keyword candidate recall). + * Extracted to a shared module to prevent implementation drift. + */ +export function extractWords(text: string): Set { + const words = new Set(); + + // Latin words (2+ chars) + const latinWords = text.toLowerCase().match(/[a-z0-9]{2,}/g); + if (latinWords) { + for (const w of latinWords) words.add(w); + } + + // CJK characters (each char as a "word", plus 2-gram) + const cjkChars = text.match(/[\u4e00-\u9fff\u3040-\u30ff\uac00-\ud7af]/g); + if (cjkChars) { + for (const c of cjkChars) words.add(c); + // 2-grams for better matching + for (let i = 0; i < cjkChars.length - 1; i++) { + words.add(cjkChars[i] + cjkChars[i + 1]); + } + } + + return words; +}