From 0d462e0b63fd2fd9985bed40fdd5510f403310cb Mon Sep 17 00:00:00 2001 From: chrishuan Date: Wed, 20 May 2026 22:58:05 +0800 Subject: [PATCH] chore: release v0.3.5 --- CHANGELOG.md | 16 + SKILL.md | 6 +- bin/export-tencent-vdb.mjs | 20 + bin/migrate-sqlite-to-tcvdb.mjs | 12 + bin/read-local-memory.mjs | 21 + openclaw.plugin.json | 2 +- package.json | 5 +- .../export-tencent-vdb/export-tencent-vdb.ts | 634 ++++++++++ scripts/export-tencent-vdb/tsconfig.json | 19 + scripts/migrate-sqlite-to-tcvdb/README.md | 239 ++++ scripts/migrate-sqlite-to-tcvdb/cli-entry.ts | 12 + .../migrate-sqlite-to-tcvdb/config-write.ts | 129 ++ .../migrate-sqlite-to-tcvdb/manifest-write.ts | 69 ++ .../node-llama-cpp.d.ts | 11 + .../sqlite-to-tcvdb.ts | 849 +++++++++++++ scripts/migrate-sqlite-to-tcvdb/tsconfig.json | 25 + .../read-local-memory/read-local-memory.ts | 1057 +++++++++++++++++ scripts/read-local-memory/tsconfig.json | 19 + src/config.ts | 2 +- 19 files changed, 3140 insertions(+), 7 deletions(-) create mode 100644 bin/export-tencent-vdb.mjs create mode 100644 bin/migrate-sqlite-to-tcvdb.mjs create mode 100644 bin/read-local-memory.mjs create mode 100644 scripts/export-tencent-vdb/export-tencent-vdb.ts create mode 100644 scripts/export-tencent-vdb/tsconfig.json create mode 100644 scripts/migrate-sqlite-to-tcvdb/README.md create mode 100644 scripts/migrate-sqlite-to-tcvdb/cli-entry.ts create mode 100644 scripts/migrate-sqlite-to-tcvdb/config-write.ts create mode 100644 scripts/migrate-sqlite-to-tcvdb/manifest-write.ts create mode 100644 scripts/migrate-sqlite-to-tcvdb/node-llama-cpp.d.ts create mode 100644 scripts/migrate-sqlite-to-tcvdb/sqlite-to-tcvdb.ts create mode 100644 scripts/migrate-sqlite-to-tcvdb/tsconfig.json create mode 100644 scripts/read-local-memory/read-local-memory.ts create mode 100644 scripts/read-local-memory/tsconfig.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f2d346..850ea71 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,22 @@ --- +## [0.3.5] - 2026-05-15 + +### 🐛 修复 + +- **兼容 OpenClaw v2026.5.7 zod v4 子路径**：显式声明 `zod@^4.4.3` 依赖，解决 `@ai-sdk/provider-utils@4.x` 需要 `zod/v4` 子路径导出但宿主环境可能 hoist zod@3.x 引发 `Cannot find module zod/v4` 的运行时错误。 + +### ✨ 改进 + +- **L1→L2 延迟从 90s 降至 10s**：`l2DelayAfterL1Seconds` 默认值 90→10，冷启动用户不再需要等待 ~90s 才能看到 L2 场景提取结果，体感更及时。 + +### 📖 文档 + +- README 新增 Docker Quick Start 章节，说明模型 URL/Name 环境变量配置方式。 + +--- + ## [0.3.4] - 2026-05-12 ### 🐛 修复 diff --git a/SKILL.md b/SKILL.md index f6bcf78..0b43367 100644 --- a/SKILL.md +++ b/SKILL.md @@ -97,7 +97,7 @@ openclaw plugins update memory-tencentdb "everyNConversations": 5, "enableWarmup": true, "l1IdleTimeoutSeconds": 600, - "l2DelayAfterL1Seconds": 90, + "l2DelayAfterL1Seconds": 10, "l2MinIntervalSeconds": 900, "l2MaxIntervalSeconds": 3600, "sessionActiveWindowHours": 24 @@ -153,7 +153,7 @@ openclaw gateway restart 检查项： - Gateway 日志中出现 `[memory-tdai]` 前缀 -- 数据目录已创建：`~/.openclaw/memory-tdai/` +- 数据目录已创建：`~/.openclaw/state/memory-tdai/` - 至少包含：`conversations/`、`records/`、`scene_blocks/`、`vectors.db` ### 7) 功能冒烟测试 @@ -198,4 +198,4 @@ openclaw gateway restart - 已完成 `memory-tencentdb` 安装与配置，并重启 Gateway。 - 已验证日志与数据目录生效，记忆链路可用。 -- 如需下一步优化，可继续调优 `recall.scoreThreshold`、`pipeline.everyNConversations`、`persona.triggerEveryN` 与 `embedding` 模型参数。 +- 如需下一步优化，可继续调优 `recall.scoreThreshold`、`pipeline.everyNConversations`、`persona.triggerEveryN` 与 `embedding` 模型参数。 \ No newline at end of file diff --git a/bin/export-tencent-vdb.mjs b/bin/export-tencent-vdb.mjs new file mode 100644 index 0000000..84c1b1d --- /dev/null +++ b/bin/export-tencent-vdb.mjs @@ -0,0 +1,20 @@ +#!/usr/bin/env node + +// 薄启动器：加载预编译好的 VDB 导出脚本。 +// 构建：npm run build:export-vdb +// 使用：npm run export:vdb -- [参数] 或 node ./bin/export-tencent-vdb.mjs [参数] + +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import fs from "node:fs"; + +const thisDir = path.dirname(fileURLToPath(import.meta.url)); +const entryScript = path.resolve(thisDir, "../scripts/export-tencent-vdb/dist/export-tencent-vdb.js"); + +if (!fs.existsSync(entryScript)) { + console.error("❌ 预编译产物不存在: " + entryScript); + console.error(" 请先执行: npm run build:export-tencent-vdb"); + process.exit(1); +} + +import(entryScript); diff --git a/bin/migrate-sqlite-to-tcvdb.mjs b/bin/migrate-sqlite-to-tcvdb.mjs new file mode 100644 index 0000000..7e02cdd --- /dev/null +++ b/bin/migrate-sqlite-to-tcvdb.mjs @@ -0,0 +1,12 @@ +#!/usr/bin/env node + +// Thin wrapper: runs the pre-compiled migration CLI entry. +// Build first: npm run build:migrate-sqlite-to-vdb (or pnpm build:migrate-sqlite-to-vdb) + +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const thisDir = path.dirname(fileURLToPath(import.meta.url)); +const entryScript = path.resolve(thisDir, "../scripts/migrate-sqlite-to-tcvdb/dist/scripts/migrate-sqlite-to-tcvdb/cli-entry.js"); + +import(entryScript); diff --git a/bin/read-local-memory.mjs b/bin/read-local-memory.mjs new file mode 100644 index 0000000..3b8ed99 --- /dev/null +++ b/bin/read-local-memory.mjs @@ -0,0 +1,21 @@ +#!/usr/bin/env node + +// 薄启动器：加载预编译好的本地 Memory 数据查询脚本。 +// 构建：npm run build:read-local-memory +// 使用：npm run read-local-memory -- [参数] 或 node ./bin/read-local-memory.mjs [参数] + +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import fs from "node:fs"; + +const thisDir = path.dirname(fileURLToPath(import.meta.url)); +const entryScript = path.resolve(thisDir, "../scripts/read-local-memory/dist/read-local-memory.js"); + +if (!fs.existsSync(entryScript)) { + console.error("❌ 预编译产物不存在: " + entryScript); + console.error(" 请先执行: npm run build:read-local-memory"); + process.exit(1); +} + +import(entryScript); diff --git a/openclaw.plugin.json b/openclaw.plugin.json index 514b5fd..f6ea5fd 100644 --- a/openclaw.plugin.json +++ b/openclaw.plugin.json @@ -57,7 +57,7 @@ "everyNConversations": { "type": "number", "default": 5, "description": "每 N 轮对话触发 L1 批处理" }, "enableWarmup": { "type": "boolean", "default": true, "description": "Warm-up 模式：新 session 从 1 轮触发开始，每次 L1 后翻倍（1→2→4→...→everyN），加速早期记忆提取" }, "l1IdleTimeoutSeconds": { "type": "number", "default": 600, "description": "L1 空闲超时（秒）：用户停止对话后多久触发 L1 批处理" }, - "l2DelayAfterL1Seconds": { "type": "number", "default": 90, "description": "L1 完成后延迟多久触发 L2（秒）" }, + "l2DelayAfterL1Seconds": { "type": "number", "default": 10, "description": "L1 完成后延迟多久触发 L2（秒）" }, "l2MinIntervalSeconds": { "type": "number", "default": 900, "description": "同一 session 两次 L2 抽取的最小间隔（秒）" }, "l2MaxIntervalSeconds": { "type": "number", "default": 3600, "description": "同一活跃 session 的 L2 最大轮询间隔（秒）" }, "sessionActiveWindowHours": { "type": "number", "default": 24, "description": "session 活跃窗口（小时），超过此时间不活跃的 session 停止 L2 轮询" } diff --git a/package.json b/package.json index 2d74158..0609611 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tencentdb-agent-memory/memory-tencentdb", - "version": "0.3.4", + "version": "0.3.5", "description": "Four-layer local memory system plugin for OpenClaw — auto-captures, structures, and profiles conversational knowledge using local LLM + SQLite vector search (L0→L1→L2→L3 pipeline)", "type": "module", "main": "./dist/index.mjs", @@ -82,7 +82,8 @@ "sqlite-vec": "0.1.7-alpha.2", "tsx": "^4.21.0", "undici": "^8.1.0", - "yaml": "^2.8.3" + "yaml": "^2.8.3", + "zod": "^4.4.3" }, "optionalDependencies": { "opik": "^1.0.0" diff --git a/scripts/export-tencent-vdb/export-tencent-vdb.ts b/scripts/export-tencent-vdb/export-tencent-vdb.ts new file mode 100644 index 0000000..d8e6259 --- /dev/null +++ b/scripts/export-tencent-vdb/export-tencent-vdb.ts @@ -0,0 +1,634 @@ +#!/usr/bin/env node +/** + * 腾讯云 VDB (Tencent VectorDB) 数据导出脚本 + * + * 连接腾讯云向量数据库实例，查询指定数据库下 collection 的文档，导出为 .jsonl 文件。 + * 仅支持腾讯云向量数据库（Tencent VectorDB），不支持其他厂商的向量数据库。 + * + * 所有连接参数通过 CLI 传入，无需 .env 文件。 + * + * 用法： + * node ./bin/export-tencent-vdb.mjs --url <地址> --username <用户名> --api-key <密钥> --database <库名> + * node ./bin/export-tencent-vdb.mjs --url <地址> --username <用户名> --api-key <密钥> --database <库名> --probe + * node ./bin/export-tencent-vdb.mjs --url <地址> --username <用户名> --api-key <密钥> --database <库名> -c -o /tmp/backup + * + * 输出： + * 默认输出到当前工作目录下的 ./vdb-export-YYYY-MM-DD/，可通过 -o 指定。 + * / + * ├── .jsonl — 每行一个 JSON 文档 + * ├── schemas.json — 导出的 collection 表结构（索引、embedding 配置等） + * └── export-meta.json — 导出元信息 + * + * 导出字段说明： + * 默认行为：导出所有字段，但跳过 vector（稠密向量，1024维浮点数组，体积大）。 + * 加 --include-vectors：导出全部字段，包括 vector，不跳过任何内容。 + * 注：sparse_vector（BM25 稀疏向量）始终导出，不受此开关影响。 + * + * 依赖：Node.js >= 18（内置 fetch） + */ + +import fs from "node:fs"; +import path from "node:path"; + +// ============================================================ +// CLI 参数解析（含 VDB 连接信息） +// ============================================================ + +interface VDBConfig { + url: string; + username: string; + apiKey: string; + database: string; + timeout: number; +} + +interface CliArgs { + // 连接参数 + url?: string; + username?: string; + apiKey?: string; + database?: string; + timeout: number; + // 导出参数 + output: string; + collection?: string; + filter?: string; + limit?: number; + offset: number; + includeVectors: boolean; + probe: boolean; + help: boolean; +} + +const PAGE_SIZE = 100; + +function parseArgs(): CliArgs { + const args = process.argv.slice(2); + const result: CliArgs = { + timeout: 30000, + output: `./vdb-export-${new Date().toISOString().slice(0, 10)}`, + offset: 0, + includeVectors: false, + probe: false, + help: false, + }; + + for (let i = 0; i < args.length; i++) { + switch (args[i]) { + case "--url": + result.url = args[++i]; + break; + case "--username": + result.username = args[++i]; + break; + case "--api-key": + result.apiKey = args[++i]; + break; + case "--database": + result.database = args[++i]; + break; + case "--timeout": + result.timeout = parseInt(args[++i], 10) || 30000; + break; + case "--output": + case "-o": + result.output = args[++i]; + break; + case "--collection": + case "-c": + result.collection = args[++i]; + break; + case "--filter": + case "-f": + result.filter = args[++i]; + break; + case "--limit": + case "-l": { + const v = parseInt(args[++i], 10); + if (isNaN(v) || v < 1) { + console.error(`❌ --limit 必须 >= 1，收到: ${args[i]}`); + process.exit(1); + } + result.limit = v; + break; + } + case "--offset": { + const v = parseInt(args[++i], 10); + if (isNaN(v) || v < 0) { + console.error(`❌ --offset 必须 >= 0，收到: ${args[i]}`); + process.exit(1); + } + result.offset = v; + break; + } + case "--include-vectors": + result.includeVectors = true; + break; + case "--probe": + result.probe = true; + break; + case "--help": + case "-h": + result.help = true; + break; + } + } + + return result; +} + +function validateConfig(args: CliArgs): VDBConfig { + const missing: string[] = []; + if (!args.url) missing.push("--url"); + if (!args.username) missing.push("--username"); + if (!args.apiKey) missing.push("--api-key"); + if (!args.database) missing.push("--database"); + + if (missing.length > 0) { + console.error("❌ 缺少必填参数："); + for (const k of missing) { + console.error(` - ${k}`); + } + console.error(); + console.error("示例："); + console.error(); + console.error(' node ./bin/export-tencent-vdb.mjs \\'); + console.error(' --url "http://your-vdb-host:8100" \\'); + console.error(' --username "root" \\'); + console.error(' --api-key "your-api-key" \\'); + console.error(' --database "your-database"'); + console.error(); + console.error("使用 --help 查看完整参数说明。"); + process.exit(1); + } + + return { + url: args.url!, + username: args.username!, + apiKey: args.apiKey!, + database: args.database!, + timeout: args.timeout, + }; +} + +function printHelp(): void { + console.log(` +腾讯云 VDB (Tencent VectorDB) 数据导出脚本 + +用法： + node ./bin/export-tencent-vdb.mjs [连接参数] [选项] + +连接参数（必填）： + --url <地址> VDB 实例 HTTP 地址（如 http://your-vdb-host:8100） + --username <用户名> 认证用户名（如 root） + --api-key <密钥> 认证密钥 + --database <库名> 数据库名称 + +选项： + --timeout <毫秒> 单次请求超时（默认: 30000） + -o, --output <目录> 输出目录（默认: ./vdb-export-YYYY-MM-DD） + -c, --collection <全名> 只导出指定 collection（全名匹配，不指定则导出所有） + -f, --filter <表达式> VDB Filter 过滤条件（如 'agent_id = "xxx"'） + -l, --limit <数量> 最多导出多少条（不指定则导出全部） + --offset <偏移> 从第几条开始（默认: 0），须为分页大小的整数倍 + --include-vectors 保留 vector 稠密向量字段（默认跳过） + --probe 仅测试连通性，列出 collection 信息后退出 + -h, --help 显示帮助 + +输出： + / + ├── .jsonl 每行一个 JSON 文档 + ├── schemas.json 表结构 + └── export-meta.json 导出元信息 + +导出字段说明： + 默认跳过 vector（稠密向量），保留 sparse_vector（BM25）。 + 加 --include-vectors 导出全部字段。 + +示例： + # 测试连通性 + node ./bin/export-tencent-vdb.mjs \\ + --url "http://gz-vdb-xxx:8100" --username root --api-key "xxx" --database mydb \\ + --probe + + # 全量导出 + node ./bin/export-tencent-vdb.mjs \\ + --url "http://gz-vdb-xxx:8100" --username root --api-key "xxx" --database mydb + + # 导出指定 collection 到指定目录 + node ./bin/export-tencent-vdb.mjs \\ + --url "http://gz-vdb-xxx:8100" --username root --api-key "xxx" --database mydb \\ + -c mydb_l0_conversations -o /tmp/backup + + # 带过滤条件 + node ./bin/export-tencent-vdb.mjs \\ + --url "http://gz-vdb-xxx:8100" --username root --api-key "xxx" --database mydb \\ + -f 'role = "user"' +`); +} + +// ============================================================ +// VDB HTTP Client +// ============================================================ + +class VDBClient { + private baseUrl: string; + private authHeader: string; + private database: string; + private timeout: number; + + constructor(cfg: VDBConfig) { + this.baseUrl = cfg.url.replace(/\/$/, ""); + this.authHeader = `Bearer account=${cfg.username}&api_key=${cfg.apiKey}`; + this.database = cfg.database; + this.timeout = cfg.timeout; + } + + async request(apiPath: string, body: Record): Promise { + const url = `${this.baseUrl}${apiPath}`; + + const resp = await fetch(url, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: this.authHeader, + }, + body: JSON.stringify(body), + signal: AbortSignal.timeout(this.timeout), + }); + + if (!resp.ok) { + const text = await resp.text().catch(() => "(unable to read body)"); + throw new Error(`VDB API error: HTTP ${resp.status} — ${text.slice(0, 500)}`); + } + + const json = (await resp.json()) as { code: number; msg: string } & T; + if (json.code !== 0) { + throw new Error(`VDB API error [${apiPath}]: code=${json.code}, msg=${json.msg}`); + } + return json; + } + + async listCollections(): Promise< + Array<{ collection: string; documentCount: number }> + > { + const result = await this.request<{ + collections: Array<{ + collection: string; + documentCount: number; + [key: string]: unknown; + }>; + }>("/collection/list", { + database: this.database, + }); + return (result.collections || []).map((c) => ({ + collection: c.collection, + documentCount: c.documentCount ?? 0, + })); + } + + async queryDocuments( + collection: string, + options: { + limit: number; + offset: number; + filter?: string; + retrieveVector?: boolean; + }, + ): Promise<{ + documents: Array>; + count: number; + }> { + const query: Record = { + limit: options.limit, + offset: options.offset, + }; + if (options.filter) { + query.filter = options.filter; + } + if (options.retrieveVector) { + query.retrieveVector = true; + } + + const result = await this.request<{ + documents: Array>; + count: number; + }>("/document/query", { + database: this.database, + collection, + readConsistency: "strongConsistency", + query, + }); + + return { + documents: result.documents || [], + count: result.count ?? 0, + }; + } + + async describeCollection(collection: string): Promise> { + const result = await this.request<{ + collection: Record; + }>("/collection/describe", { + database: this.database, + collection, + }); + return result.collection || {}; + } +} + +// ============================================================ +// 导出逻辑 +// ============================================================ + +interface ExportOptions { + filter?: string; + limit?: number; + offset: number; + includeVectors: boolean; + expectedTotal?: number; +} + +async function exportCollection( + client: VDBClient, + collection: string, + outputDir: string, + options: ExportOptions, +): Promise<{ docCount: number; filePath: string }> { + const filePath = path.join(outputDir, `${collection}.jsonl`); + const writeStream = fs.createWriteStream(filePath, { encoding: "utf-8" }); + + const isRangeMode = options.limit !== undefined; + const maxDocs = options.limit ?? Infinity; + const pageSize = isRangeMode ? Math.min(options.limit!, PAGE_SIZE) : PAGE_SIZE; + + let currentOffset = options.offset; + let totalExported = 0; + let hasMore = true; + + console.log(` 📦 ${collection}`); + if (options.expectedTotal !== undefined) { + console.log(` 文档总数: ${options.expectedTotal}`); + } + if (options.filter) { + console.log(` 过滤条件: ${options.filter}`); + } + if (isRangeMode) { + console.log(` 导出范围: offset=${options.offset}, limit=${options.limit}`); + } + + while (hasMore && totalExported < maxDocs) { + const remaining = maxDocs - totalExported; + const thisPageSize = Math.min(pageSize, remaining); + + try { + const result = await client.queryDocuments(collection, { + limit: thisPageSize, + offset: currentOffset, + filter: options.filter, + retrieveVector: options.includeVectors, + }); + + const docs = result.documents; + if (!docs || docs.length === 0) { + hasMore = false; + break; + } + + for (const doc of docs) { + const exportDoc = { ...doc }; + if (!options.includeVectors) { + delete exportDoc.vector; + } + writeStream.write(JSON.stringify(exportDoc) + "\n"); + } + + totalExported += docs.length; + currentOffset += docs.length; + + if (options.expectedTotal !== undefined && !isRangeMode) { + const pct = Math.min( + 100, + Math.round((totalExported / options.expectedTotal) * 100), + ); + process.stdout.write( + `\r 进度: ${totalExported}/${options.expectedTotal} (${pct}%)`, + ); + } else { + process.stdout.write(`\r 已导出: ${totalExported} 条`); + } + + if (docs.length < thisPageSize) { + hasMore = false; + } + } catch (err) { + console.error( + `\n ❌ 查询失败 (offset=${currentOffset}): ${err instanceof Error ? err.message : String(err)}`, + ); + hasMore = false; + } + } + + writeStream.end(); + await new Promise((resolve) => writeStream.on("finish", resolve)); + + console.log( + `\n ✅ 完成: ${totalExported} 条 → ${path.basename(filePath)}`, + ); + + return { docCount: totalExported, filePath }; +} + +// ============================================================ +// Main +// ============================================================ + +async function main(): Promise { + const args = parseArgs(); + + if (args.help) { + printHelp(); + process.exit(0); + } + + const config = validateConfig(args); + + console.log("╔═══════════════════════════════════════════════════╗"); + console.log("║ 腾讯云 VDB (Tencent VectorDB) 数据导出工具 ║"); + console.log("╚═══════════════════════════════════════════════════╝"); + console.log(); + console.log(`📌 VDB 地址: ${config.url}`); + console.log(`📌 数据库: ${config.database}`); + console.log(`📌 输出目录: ${args.output}`); + if (args.collection) { + console.log(`📌 指定导出: ${args.collection}`); + } + if (args.filter) { + console.log(`📌 过滤条件: ${args.filter}`); + } + if (args.limit !== undefined) { + console.log(`📌 导出上限: ${args.limit} 条`); + } + if (args.offset > 0) { + console.log(`📌 起始偏移: ${args.offset}`); + } + if (args.includeVectors) { + console.log(`📌 包含向量: 是`); + } + console.log(); + + fs.mkdirSync(args.output, { recursive: true }); + + const client = new VDBClient(config); + + let allCollections: Array<{ collection: string; documentCount: number }>; + try { + allCollections = await client.listCollections(); + } catch (err) { + console.error( + `❌ 列出 collection 失败: ${err instanceof Error ? err.message : String(err)}`, + ); + process.exit(1); + } + + let targetCollections: Array<{ collection: string; documentCount: number }>; + if (args.collection) { + const found = allCollections.find((c) => c.collection === args.collection); + if (!found) { + console.error( + `❌ Collection "${args.collection}" 不存在。可用的 collection：`, + ); + for (const c of allCollections) { + console.error(` - ${c.collection} (${c.documentCount} 条)`); + } + process.exit(1); + } + targetCollections = [found]; + } else { + targetCollections = allCollections; + console.log( + `🔍 找到 ${targetCollections.length} 个 collection：`, + ); + for (const c of targetCollections) { + console.log(` - ${c.collection} (${c.documentCount} 条)`); + } + } + + if (targetCollections.length === 0) { + console.log("⚠️ 数据库中没有 collection，无数据可导出"); + process.exit(0); + } + + // --probe 模式：只测试连通性，列出信息后退出 + if (args.probe) { + console.log(); + console.log("✅ 连通性测试通过"); + console.log(); + console.log(` VDB 地址: ${config.url}`); + console.log(` 数据库: ${config.database}`); + console.log(` Collection: ${targetCollections.length} 个`); + const totalDocs = targetCollections.reduce((s, c) => s + c.documentCount, 0); + console.log(` 总文档数: ${totalDocs}`); + console.log(); + for (const c of targetCollections) { + console.log(` - ${c.collection} (${c.documentCount} 条)`); + } + console.log(); + process.exit(0); + } + + console.log(); + + // 获取并保存表结构 + const schemas: Record> = {}; + console.log("📐 获取表结构..."); + for (const col of targetCollections) { + try { + const schema = await client.describeCollection(col.collection); + schemas[col.collection] = schema; + const indexCount = Array.isArray(schema.indexes) ? schema.indexes.length : 0; + const emb = schema.embedding as Record | undefined; + const embInfo = emb ? `embedding=${emb.field}→${emb.model}` : "无 embedding"; + console.log(` ✅ ${col.collection} (${indexCount} 个索引, ${embInfo})`); + } catch (err) { + console.error( + ` ⚠️ ${col.collection} 表结构获取失败: ${err instanceof Error ? err.message : String(err)}`, + ); + } + } + console.log(); + + const schemaPath = path.join(args.output, "schemas.json"); + fs.writeFileSync(schemaPath, JSON.stringify(schemas, null, 2) + "\n"); + + const exportResults: Array<{ + collection: string; + docCount: number; + filePath: string; + }> = []; + + for (const col of targetCollections) { + try { + const result = await exportCollection(client, col.collection, args.output, { + filter: args.filter, + limit: args.limit, + offset: args.offset, + includeVectors: args.includeVectors, + expectedTotal: col.documentCount, + }); + exportResults.push({ collection: col.collection, ...result }); + } catch (err) { + console.error( + `❌ 导出 ${col.collection} 失败: ${err instanceof Error ? err.message : String(err)}`, + ); + exportResults.push({ + collection: col.collection, + docCount: 0, + filePath: "", + }); + } + console.log(); + } + + const meta = { + exportedAt: new Date().toISOString(), + vdbUrl: config.url, + database: config.database, + filter: args.filter ?? null, + offset: args.offset, + limit: args.limit ?? null, + includeVectors: args.includeVectors, + collections: exportResults.map((r) => ({ + collection: r.collection, + documentCount: r.docCount, + file: r.filePath ? path.basename(r.filePath) : null, + })), + totalDocuments: exportResults.reduce((sum, r) => sum + r.docCount, 0), + }; + + const metaPath = path.join(args.output, "export-meta.json"); + fs.writeFileSync(metaPath, JSON.stringify(meta, null, 2) + "\n"); + + console.log("═══════════════════════════════════════════════════"); + console.log(" ✅ 导出完成"); + console.log("═══════════════════════════════════════════════════"); + console.log(); + console.log(` 📁 输出目录: ${args.output}`); + console.log(` 📊 总文档数: ${meta.totalDocuments}`); + for (const r of exportResults) { + const status = r.docCount > 0 ? "✅" : "⚠️"; + console.log( + ` ${status} ${r.collection}: ${r.docCount} 条`, + ); + } + console.log(` 📋 元信息: ${path.basename(metaPath)}`); + console.log(` 📐 表结构: ${path.basename(schemaPath)}`); + console.log(); +} + +main().catch((err) => { + console.error( + `\n❌ 导出失败: ${err instanceof Error ? err.message : String(err)}`, + ); + process.exit(1); +}); diff --git a/scripts/export-tencent-vdb/tsconfig.json b/scripts/export-tencent-vdb/tsconfig.json new file mode 100644 index 0000000..9a0a230 --- /dev/null +++ b/scripts/export-tencent-vdb/tsconfig.json @@ -0,0 +1,19 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "Node16", + "moduleResolution": "Node16", + "outDir": "./dist", + "rootDir": ".", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "resolveJsonModule": true, + "types": ["node"], + "declaration": false, + "sourceMap": false + }, + "include": ["export-tencent-vdb.ts"], + "exclude": ["dist", "node_modules", "docs"] +} diff --git a/scripts/migrate-sqlite-to-tcvdb/README.md b/scripts/migrate-sqlite-to-tcvdb/README.md new file mode 100644 index 0000000..a0c6aad --- /dev/null +++ b/scripts/migrate-sqlite-to-tcvdb/README.md @@ -0,0 +1,239 @@ +# SQLite → 腾讯云向量数据库迁移工具 + +离线迁移工具，用于将 memory-tdai 的数据从本地 SQLite 存储迁移到腾讯云向量数据库（TCVDB）。 + +## 前置条件 + +- Node.js >= 22.16.0 +- 插件已通过 `openclaw plugins install` 安装 +- 迁移脚本已编译（见下方） + +## 编译 + +迁移脚本使用 TypeScript 编写，运行前需要先编译： + +```bash +npm run build:migrate-sqlite-to-vdb +``` + +编译产物输出到 `scripts/migrate-sqlite-to-tcvdb/dist/`，可直接用 Node 运行。 + +## 使用方法 + +```bash +# 预检模式（仅查看源数据，不执行写入） +npm run migrate:sqlite-to-tcvdb -- \ + --plugin-data-dir ~/.openclaw/memory-tdai \ + --openclaw-config-path ~/.openclaw/openclaw.json \ + --tcvdb-url http://127.0.0.1:80 \ + --tcvdb-username root \ + --tcvdb-api-key-env TCVDB_API_KEY \ + --tcvdb-database agent_memory_prod \ + --tcvdb-embedding-model bge-large-zh \ + --dry-run + +# 正式迁移 +npm run migrate:sqlite-to-tcvdb -- \ + --plugin-data-dir ~/.openclaw/memory-tdai \ + --openclaw-config-path ~/.openclaw/openclaw.json \ + --tcvdb-url http://127.0.0.1:80 \ + --tcvdb-username root \ + --tcvdb-api-key-env TCVDB_API_KEY \ + --tcvdb-database agent_memory_prod \ + --tcvdb-embedding-model bge-large-zh \ + --yes +``` + +### 更多示例 + +```bash +# 直接传入 API Key（不通过环境变量） +npm run migrate:sqlite-to-tcvdb -- \ + --plugin-data-dir ~/.openclaw/memory-tdai \ + --openclaw-config-path ~/.openclaw/openclaw.json \ + --tcvdb-url http://127.0.0.1:80 \ + --tcvdb-username root \ + --tcvdb-api-key 'your-api-key-here' \ + --tcvdb-database agent_memory_prod \ + --tcvdb-embedding-model bge-large-zh \ + --yes +``` + +```bash +# 指定自定义 SQLite 路径（数据库不在默认 vectors.db 位置时） +npm run migrate:sqlite-to-tcvdb -- \ + --plugin-data-dir ~/.openclaw/memory-tdai \ + --sqlite-path /backup/2026-04/vectors-snapshot.db \ + --openclaw-config-path ~/.openclaw/openclaw.json \ + --tcvdb-url http://127.0.0.1:80 \ + --tcvdb-username root \ + --tcvdb-api-key-env TCVDB_API_KEY \ + --tcvdb-database agent_memory_prod \ + --tcvdb-embedding-model bge-large-zh \ + --yes +``` + +```bash +# 只迁移 L1 记忆层（跳过 L0 原始消息和 Profile） +npm run migrate:sqlite-to-tcvdb -- \ + --plugin-data-dir ~/.openclaw/memory-tdai \ + --openclaw-config-path ~/.openclaw/openclaw.json \ + --tcvdb-url http://127.0.0.1:80 \ + --tcvdb-username root \ + --tcvdb-api-key-env TCVDB_API_KEY \ + --tcvdb-database agent_memory_prod \ + --tcvdb-embedding-model bge-large-zh \ + --layers l1 \ + --yes +``` + +```bash +# 只迁移 L0 和 L1（不迁移 Profile） +npm run migrate:sqlite-to-tcvdb -- \ + --plugin-data-dir ~/.openclaw/memory-tdai \ + --openclaw-config-path ~/.openclaw/openclaw.json \ + --tcvdb-url http://127.0.0.1:80 \ + --tcvdb-username root \ + --tcvdb-api-key-env TCVDB_API_KEY \ + --tcvdb-database agent_memory_prod \ + --tcvdb-embedding-model bge-large-zh \ + --layers l0,l1 \ + --yes +``` + +```bash +# 英文语料场景：使用英文 BM25 分词 +npm run migrate:sqlite-to-tcvdb -- \ + --plugin-data-dir ~/.openclaw/memory-tdai \ + --openclaw-config-path ~/.openclaw/openclaw.json \ + --tcvdb-url http://127.0.0.1:80 \ + --tcvdb-username root \ + --tcvdb-api-key-env TCVDB_API_KEY \ + --tcvdb-database agent_memory_prod \ + --tcvdb-embedding-model bge-large-en-v1.5 \ + --bm25-language en \ + --yes +``` + +```bash +# 禁用 BM25 稀疏向量（仅使用密集向量检索） +npm run migrate:sqlite-to-tcvdb -- \ + --plugin-data-dir ~/.openclaw/memory-tdai \ + --openclaw-config-path ~/.openclaw/openclaw.json \ + --tcvdb-url http://127.0.0.1:80 \ + --tcvdb-username root \ + --tcvdb-api-key-env TCVDB_API_KEY \ + --tcvdb-database agent_memory_prod \ + --tcvdb-embedding-model bge-large-zh \ + --no-bm25-enabled \ + --yes +``` + +```bash +# 仅迁移数据，不自动更新 openclaw.json 和 manifest（手动管理配置） +npm run migrate:sqlite-to-tcvdb -- \ + --plugin-data-dir ~/.openclaw/memory-tdai \ + --openclaw-config-path ~/.openclaw/openclaw.json \ + --tcvdb-url http://127.0.0.1:80 \ + --tcvdb-username root \ + --tcvdb-api-key-env TCVDB_API_KEY \ + --tcvdb-database agent_memory_prod \ + --tcvdb-embedding-model bge-large-zh \ + --no-apply-config \ + --no-rewrite-manifest \ + --yes +``` + +```bash +# 追加迁移：允许目标库已有数据，跳过非空检查 +npm run migrate:sqlite-to-tcvdb -- \ + --plugin-data-dir ~/.openclaw/memory-tdai \ + --openclaw-config-path ~/.openclaw/openclaw.json \ + --tcvdb-url http://127.0.0.1:80 \ + --tcvdb-username root \ + --tcvdb-api-key-env TCVDB_API_KEY \ + --tcvdb-database agent_memory_prod \ + --tcvdb-embedding-model bge-large-zh \ + --no-fail-if-target-nonempty \ + --no-verify-counts \ + --yes +``` + +```bash +# 输出迁移摘要到 JSON 文件（适合 CI/自动化流水线） +npm run migrate:sqlite-to-tcvdb -- \ + --plugin-data-dir ~/.openclaw/memory-tdai \ + --openclaw-config-path ~/.openclaw/openclaw.json \ + --tcvdb-url http://127.0.0.1:80 \ + --tcvdb-username root \ + --tcvdb-api-key-env TCVDB_API_KEY \ + --tcvdb-database agent_memory_prod \ + --tcvdb-embedding-model bge-large-zh \ + --summary-json-path ./migration-report.json \ + --job-id "migrate-2026-04-13" \ + --yes +``` + +```bash +# 设置自定义超时和别名 +npm run migrate:sqlite-to-tcvdb -- \ + --plugin-data-dir ~/.openclaw/memory-tdai \ + --openclaw-config-path ~/.openclaw/openclaw.json \ + --tcvdb-url http://10.0.1.50:80 \ + --tcvdb-username admin \ + --tcvdb-api-key-env TCVDB_API_KEY \ + --tcvdb-database agent_memory_prod \ + --tcvdb-embedding-model bge-large-zh \ + --tcvdb-alias "生产环境-主库" \ + --tcvdb-timeout-ms 30000 \ + --yes +``` + +## 参数说明 + +| 参数 | 必填 | 默认值 | 说明 | +|---|---|---|---| +| `--plugin-data-dir` | 是 | — | 插件数据目录路径 | +| `--openclaw-config-path` | 是 | — | `openclaw.json` 配置文件路径 | +| `--sqlite-path` | 否 | `/vectors.db` | SQLite 数据库文件路径（默认取数据目录下的 `vectors.db`） | +| `--plugin-id` | 否 | `memory-tencentdb` | 写入配置时使用的插件 ID | +| `--tcvdb-url` | 是 | — | TCVDB 服务地址 | +| `--tcvdb-username` | 是 | — | TCVDB 用户名 | +| `--tcvdb-api-key` | * | — | TCVDB API 密钥（明文） | +| `--tcvdb-api-key-env` | * | — | 包含 API 密钥的环境变量名 | +| `--tcvdb-database` | 是 | — | TCVDB 数据库名 | +| `--tcvdb-embedding-model` | 是 | — | Embedding 模型名称 | +| `--tcvdb-alias` | 否 | `""` | 用户自定义别名 | +| `--tcvdb-timeout-ms` | 否 | `10000` | 请求超时时间（毫秒） | +| `--layers` | 否 | `l0,l1,l2,l3` | 要迁移的层（逗号分隔） | +| `--dry-run` | 否 | `false` | 仅预览，不执行写入 | +| `--yes` | 否 | `false` | 跳过交互确认 | +| `--apply-config` | 否 | `true` | 迁移后更新 openclaw.json | +| `--config-backup` | 否 | `true` | 写入配置前先备份原配置文件 | +| `--rewrite-manifest` | 否 | `true` | 将 manifest.json 更新为 tcvdb | +| `--fail-if-target-nonempty` | 否 | `true` | 目标库非空时中止 | +| `--verify-counts` | 否 | `true` | 迁移后校验记录数 | +| `--summary-json-path` | 否 | — | 将迁移摘要写入此文件 | +| `--job-id` | 否 | — | 迁移任务 ID（用于追踪） | +| `--bm25-enabled` | 否 | `true` | 启用 BM25 稀疏向量 | +| `--bm25-language` | 否 | `zh` | BM25 语言（`zh` 或 `en`） | + +\* `--tcvdb-api-key` 和 `--tcvdb-api-key-env` 二选一，必须提供其中一个。 + +## 目录结构 + +``` +scripts/migrate-sqlite-to-tcvdb/ +├── cli-entry.ts # CLI 入口 +├── sqlite-to-tcvdb.ts # 迁移核心逻辑（参数解析、预检、数据迁移） +├── config-write.ts # OpenClaw 配置更新（JSON5，自包含） +├── manifest-write.ts # Manifest 重写 +├── *.test.ts # 就近放置的测试文件 +├── tsconfig.json # 迁移脚本编译配置 +├── dist/ # 编译产物（已 gitignore） +└── README.md # 本文件 + +bin/migrate-sqlite-to-tcvdb.mjs # 极薄 bin 包装 → dist/ +``` + +迁移脚本通过 `../../src/` 引用存储实现（VectorStore、TcvdbMemoryStore 等），但**不依赖 `openclaw/plugin-sdk`**。配置写回直接使用 `json5`。 diff --git a/scripts/migrate-sqlite-to-tcvdb/cli-entry.ts b/scripts/migrate-sqlite-to-tcvdb/cli-entry.ts new file mode 100644 index 0000000..82ba51f --- /dev/null +++ b/scripts/migrate-sqlite-to-tcvdb/cli-entry.ts @@ -0,0 +1,12 @@ +import { runMigrationCli } from "./sqlite-to-tcvdb.js"; + +const TAG = "[memory-tdai][migrate-cli]"; + +try { + const summary = await runMigrationCli(process.argv.slice(2)); + process.stdout.write(`${JSON.stringify(summary, null, 2)}\n`); +} catch (err) { + const message = err instanceof Error ? err.message : String(err); + process.stderr.write(`${TAG} ${message}\n`); + process.exitCode = 1; +} diff --git a/scripts/migrate-sqlite-to-tcvdb/config-write.ts b/scripts/migrate-sqlite-to-tcvdb/config-write.ts new file mode 100644 index 0000000..9e89fe8 --- /dev/null +++ b/scripts/migrate-sqlite-to-tcvdb/config-write.ts @@ -0,0 +1,129 @@ +import fs from "node:fs/promises"; +import JSON5 from "json5"; + +export type Bm25Language = "zh" | "en"; + +export interface MigrationPluginConfigTarget { + url: string; + username: string; + apiKey: string; + database: string; + alias: string; + embeddingModel: string; + timeout: number; + bm25Enabled: boolean; + bm25Language: Bm25Language; +} + +export interface WriteMigrationPluginConfigParams { + configPath: string; + pluginId: string; + tcvdb: { + url: string; + username: string; + apiKey: string; + database: string; + alias: string; + embeddingModel: string; + timeout: number; + }; + bm25: { + enabled: boolean; + language: Bm25Language; + }; +} + +interface ConfigWriteAdapterDeps { + fs?: Pick; + parseConfig?: (raw: string) => unknown; +} + +function asRecord(value: unknown): Record { + return value !== null && typeof value === "object" && !Array.isArray(value) + ? { ...(value as Record) } + : {}; +} + +export function buildMigrationPluginConfigPatch(target: MigrationPluginConfigTarget): Record { + return { + storeBackend: "tcvdb", + tcvdb: { + url: target.url, + username: target.username, + apiKey: target.apiKey, + database: target.database, + alias: target.alias, + embeddingModel: target.embeddingModel, + timeout: target.timeout, + }, + bm25: { + enabled: target.bm25Enabled, + language: target.bm25Language, + }, + }; +} + +function applyPluginConfigPatch( + sourceConfig: Record, + pluginId: string, + patch: Record, +): Record { + const nextConfig = asRecord(sourceConfig); + const plugins = asRecord(nextConfig.plugins); + const entries = asRecord(plugins.entries); + const targetEntry = asRecord(entries[pluginId]); + const targetPluginConfig = asRecord(targetEntry.config); + const patchTcvdb = asRecord(patch.tcvdb); + const patchBm25 = asRecord(patch.bm25); + + const mergedPluginConfig: Record = { + ...targetPluginConfig, + ...patch, + tcvdb: { + ...asRecord(targetPluginConfig.tcvdb), + ...patchTcvdb, + }, + bm25: { + ...asRecord(targetPluginConfig.bm25), + ...patchBm25, + }, + }; + + entries[pluginId] = { + ...targetEntry, + config: mergedPluginConfig, + }; + + plugins.entries = entries; + nextConfig.plugins = plugins; + return nextConfig; +} + +export async function writeMigrationPluginConfig( + params: WriteMigrationPluginConfigParams, + deps: ConfigWriteAdapterDeps = {}, +): Promise { + const fsImpl = deps.fs ?? fs; + const parseConfig = deps.parseConfig ?? ((raw: string) => JSON5.parse(raw)); + + let parsed: unknown; + try { + parsed = parseConfig(await fsImpl.readFile(params.configPath, "utf-8")); + } catch { + throw new Error( + `Config migration writer only supports single-file JSON/JSON5: ${params.configPath}`, + ); + } + + const nextConfig = applyPluginConfigPatch( + asRecord(parsed), + params.pluginId, + buildMigrationPluginConfigPatch({ + ...params.tcvdb, + bm25Enabled: params.bm25.enabled, + bm25Language: params.bm25.language, + }), + ); + + await fsImpl.writeFile(params.configPath, `${JSON.stringify(nextConfig, null, 2)}\n`, "utf-8"); +} diff --git a/scripts/migrate-sqlite-to-tcvdb/manifest-write.ts b/scripts/migrate-sqlite-to-tcvdb/manifest-write.ts new file mode 100644 index 0000000..bd026b4 --- /dev/null +++ b/scripts/migrate-sqlite-to-tcvdb/manifest-write.ts @@ -0,0 +1,69 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { + buildStoreInfo, + manifestPath, + readManifest, + writeManifest, +} from "../../src/utils/manifest.js"; + +export interface RewriteMigrationManifestParams { + dataDir: string; + tcvdbUrl: string; + tcvdbDatabase: string; + tcvdbAlias?: string; + backupExisting?: boolean; + now?: () => string; +} + +export interface RewriteMigrationManifestResult { + created: boolean; + updated: boolean; + backupPath?: string; +} + +const DEFAULT_BACKUP_FILE = "manifest.json.migrate.bak"; + +export async function rewriteMigrationManifest( + params: RewriteMigrationManifestParams, +): Promise { + const existing = readManifest(params.dataDir); + const nextStore = buildStoreInfo({ + type: "tcvdb", + tcvdbUrl: params.tcvdbUrl, + tcvdbDatabase: params.tcvdbDatabase, + tcvdbAlias: params.tcvdbAlias, + }); + + if (!existing) { + writeManifest(params.dataDir, { + version: 1, + createdAt: params.now?.() ?? new Date().toISOString(), + store: nextStore, + seed: null, + }); + return { + created: true, + updated: false, + backupPath: undefined, + }; + } + + let backupPath: string | undefined; + if (params.backupExisting !== false) { + backupPath = path.join(path.dirname(manifestPath(params.dataDir)), DEFAULT_BACKUP_FILE); + await fs.mkdir(path.dirname(backupPath), { recursive: true }); + await fs.copyFile(manifestPath(params.dataDir), backupPath); + } + + writeManifest(params.dataDir, { + ...existing, + store: nextStore, + }); + + return { + created: false, + updated: true, + backupPath, + }; +} diff --git a/scripts/migrate-sqlite-to-tcvdb/node-llama-cpp.d.ts b/scripts/migrate-sqlite-to-tcvdb/node-llama-cpp.d.ts new file mode 100644 index 0000000..4dcafef --- /dev/null +++ b/scripts/migrate-sqlite-to-tcvdb/node-llama-cpp.d.ts @@ -0,0 +1,11 @@ +/** + * Type stub for node-llama-cpp — the migration script does not use local + * embedding but TypeScript still resolves the module through transitive + * imports (sqlite.ts → embedding.ts → import("node-llama-cpp")). + * + * This stub satisfies the compiler without requiring the actual package. + */ +declare module "node-llama-cpp" { + const _: any; + export = _; +} diff --git a/scripts/migrate-sqlite-to-tcvdb/sqlite-to-tcvdb.ts b/scripts/migrate-sqlite-to-tcvdb/sqlite-to-tcvdb.ts new file mode 100644 index 0000000..25f9765 --- /dev/null +++ b/scripts/migrate-sqlite-to-tcvdb/sqlite-to-tcvdb.ts @@ -0,0 +1,849 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { parseArgs } from "node:util"; +import type { MemoryRecord } from "../../src/core/record/l1-writer.js"; +import { listLocalProfiles } from "../../src/core/profile/profile-sync.js"; +import { createBM25Encoder } from "../../src/core/store/bm25-local.js"; +import { VectorStore, type L0RecordRow, type L1RecordRow } from "../../src/core/store/sqlite.js"; +import { TcvdbMemoryStore } from "../../src/core/store/tcvdb.js"; +import type { L0Record, ProfileRecord, ProfileSyncRecord, StoreInitResult } from "../../src/core/store/types.js"; +import { readManifest } from "../../src/utils/manifest.js"; +import { + rewriteMigrationManifest as rewriteMigrationManifestDefault, + type RewriteMigrationManifestResult, +} from "./manifest-write.js"; +import { + writeMigrationPluginConfig as writeMigrationPluginConfigDefault, +} from "./config-write.js"; + +export const DEFAULT_MIGRATION_PLUGIN_ID = "memory-tencentdb"; +export const ALL_MIGRATION_LAYERS = ["l0", "l1", "l2", "l3"] as const; + +const TAG = "[memory-tdai][migrate]"; + +function log(message: string): void { + process.stderr.write(`${TAG} ${message}\n`); +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +export type MigrationLayer = (typeof ALL_MIGRATION_LAYERS)[number]; +export type Bm25Language = "zh" | "en"; + +export interface ResolvedMigrationCliOptions { + pluginDataDir: string; + sqlitePath: string; + openclawConfigPath: string; + pluginId: string; + layers: MigrationLayer[]; + applyConfig: boolean; + configBackup: boolean; + rewriteManifest: boolean; + failIfTargetNonempty: boolean; + verifyCounts: boolean; + dryRun: boolean; + yes: boolean; + bm25Enabled: boolean; + bm25Language: Bm25Language; + summaryJsonPath?: string; + jobId?: string; + tcvdb: { + url: string; + username: string; + apiKey: string; + database: string; + alias: string; + embeddingModel: string; + timeout: number; + caPemPath?: string; + }; +} + +export interface MigrationPreflightSummary { + pluginId: string; + dryRun: boolean; + layers: MigrationLayer[]; + paths: { + pluginDataDir: string; + sqlitePath: string; + openclawConfigPath: string; + }; + source: { + l0Count: number; + l1Count: number; + profileCount: number; + manifestExists: boolean; + manifestStoreType: "sqlite" | "tcvdb" | null; + }; + target: { + url: string; + username: string; + database: string; + alias: string; + embeddingModel: string; + timeout: number; + bm25Enabled: boolean; + bm25Language: Bm25Language; + }; + options: { + applyConfig: boolean; + configBackup: boolean; + rewriteManifest: boolean; + failIfTargetNonempty: boolean; + verifyCounts: boolean; + yes: boolean; + }; + migration?: { + l0Migrated: number; + l1Migrated: number; + profileMigrated: number; + targetL0Count: number; + targetL1Count: number; + targetProfileCount: number; + configWritten: boolean; + manifestWritten: boolean; + manifestBackupPath?: string; + }; +} + +export const DEFAULT_MIGRATION_PAGE_SIZE = 50; + +export interface MigrationTargetStore { + init(providerInfo?: unknown): Promise | StoreInitResult; + isDegraded(): boolean; + close(): void; + upsertL1(record: MemoryRecord, embedding?: Float32Array): Promise | boolean; + upsertL0(record: L0Record, embedding?: Float32Array): Promise | boolean; + upsertL1Batch?(records: MemoryRecord[]): Promise; + upsertL0Batch?(records: L0Record[]): Promise; + countL1(): Promise | number; + countL0(): Promise | number; + pullProfiles?(): Promise; + syncProfiles?(records: ProfileSyncRecord[]): Promise; +} + +export interface RunMigrationCliDeps { + createTargetStore?: (options: ResolvedMigrationCliOptions) => MigrationTargetStore; + writeMigrationPluginConfig?: typeof writeMigrationPluginConfigDefault; + rewriteMigrationManifest?: (params: { + dataDir: string; + tcvdbUrl: string; + tcvdbDatabase: string; + tcvdbAlias?: string; + }) => Promise; + verifyDelayMs?: number; +} + +function getRequiredString(values: Record, key: string): string { + const value = values[key]; + if (typeof value !== "string" || value.trim() === "") { + throw new Error(`Missing required option --${key}`); + } + return value.trim(); +} + +function getOptionalString(values: Record, key: string): string | undefined { + const value = values[key]; + if (typeof value !== "string") return undefined; + const trimmed = value.trim(); + return trimmed === "" ? undefined : trimmed; +} + +function resolveBooleanOption( + values: Record, + key: string, + defaultValue: boolean, +): boolean { + const value = values[key]; + return typeof value === "boolean" ? value : defaultValue; +} + +function resolveTcvdbApiKey(values: Record): string { + const directApiKey = getOptionalString(values, "tcvdb-api-key"); + const apiKeyEnvName = getOptionalString(values, "tcvdb-api-key-env"); + + if (directApiKey && apiKeyEnvName) { + throw new Error("Provide either --tcvdb-api-key or --tcvdb-api-key-env, not both"); + } + if (directApiKey) { + return directApiKey; + } + if (!apiKeyEnvName) { + throw new Error("Missing required TCVDB API key input: use --tcvdb-api-key or --tcvdb-api-key-env"); + } + + const envValue = process.env[apiKeyEnvName]?.trim(); + if (!envValue) { + throw new Error(`Environment variable ${apiKeyEnvName} is empty or not set`); + } + return envValue; +} + +function parseLayers(rawLayers: string | undefined): MigrationLayer[] { + if (!rawLayers) return [...ALL_MIGRATION_LAYERS]; + + const values = rawLayers + .split(",") + .map((layer) => layer.trim()) + .filter(Boolean); + + if (values.length === 0) { + throw new Error("--layers must include at least one layer"); + } + + const uniqueLayers = [...new Set(values)]; + const invalid = uniqueLayers.filter( + (layer): layer is string => !ALL_MIGRATION_LAYERS.includes(layer as MigrationLayer), + ); + if (invalid.length > 0) { + throw new Error(`Unsupported layer(s): ${invalid.join(", ")}`); + } + + return uniqueLayers as MigrationLayer[]; +} + +function parseBm25Language(rawLanguage: string | undefined): Bm25Language { + if (!rawLanguage) return "zh"; + if (rawLanguage === "zh" || rawLanguage === "en") { + return rawLanguage; + } + throw new Error(`Unsupported --bm25-language value: ${rawLanguage}`); +} + +function parseTimeout(rawValue: string | undefined): number { + if (!rawValue) return 10000; + const timeout = Number(rawValue); + if (!Number.isFinite(timeout) || timeout <= 0) { + throw new Error(`Invalid --tcvdb-timeout-ms value: ${rawValue}`); + } + return timeout; +} + +/** + * Build a "nothing to migrate" summary — used when source data dir or sqlite + * file doesn't exist (e.g. fresh deployment that hasn't captured any data yet). + */ +function buildEmptySummary(options: ResolvedMigrationCliOptions): MigrationPreflightSummary { + return { + pluginId: options.pluginId, + dryRun: options.dryRun, + layers: options.layers, + paths: { + pluginDataDir: options.pluginDataDir, + sqlitePath: options.sqlitePath, + openclawConfigPath: options.openclawConfigPath, + }, + source: { + l0Count: 0, + l1Count: 0, + profileCount: 0, + manifestExists: false, + manifestStoreType: null, + }, + target: { + url: options.tcvdb.url, + username: options.tcvdb.username, + database: options.tcvdb.database, + alias: options.tcvdb.alias, + embeddingModel: options.tcvdb.embeddingModel, + timeout: options.tcvdb.timeout, + bm25Enabled: options.bm25Enabled, + bm25Language: options.bm25Language, + }, + options: { + applyConfig: options.applyConfig, + configBackup: options.configBackup, + rewriteManifest: options.rewriteManifest, + failIfTargetNonempty: options.failIfTargetNonempty, + verifyCounts: options.verifyCounts, + yes: options.yes, + }, + }; +} + +async function ensureReadablePath(filePath: string, label: string): Promise { + try { + await fs.access(filePath); + } catch { + throw new Error(`${label} does not exist or is not accessible: ${filePath}`); + } +} + +async function ensureReadableDirectory(dirPath: string, label: string): Promise { + const stat = await fs.stat(dirPath).catch(() => null); + if (!stat?.isDirectory()) { + throw new Error(`${label} is not a directory: ${dirPath}`); + } +} + +function safeParseMetadata(raw: string): Record { + if (!raw) return {}; + try { + const parsed = JSON.parse(raw); + return parsed && typeof parsed === "object" && !Array.isArray(parsed) + ? (parsed as Record) + : {}; + } catch { + return {}; + } +} + +function compactTimestamps(row: L1RecordRow): string[] { + return [...new Set([row.timestamp_str, row.timestamp_start, row.timestamp_end].filter(Boolean))]; +} + +function mapL1RowToMemoryRecord(row: L1RecordRow): MemoryRecord { + const timestamps = compactTimestamps(row); + const fallbackIso = row.updated_time || row.created_time || row.timestamp_end || row.timestamp_str || new Date(0).toISOString(); + return { + id: row.record_id, + content: row.content, + type: row.type as MemoryRecord["type"], + priority: row.priority, + scene_name: row.scene_name, + source_message_ids: [], + metadata: safeParseMetadata(row.metadata_json), + timestamps, + createdAt: row.created_time || fallbackIso, + updatedAt: row.updated_time || row.created_time || fallbackIso, + sessionKey: row.session_key || "", + sessionId: row.session_id || "", + }; +} + +function mapL0RowToRecord(row: L0RecordRow): L0Record { + return { + id: row.record_id, + sessionKey: row.session_key, + sessionId: row.session_id || "", + role: row.role, + messageText: row.message_text, + recordedAt: row.recorded_at || "", + timestamp: row.timestamp ?? 0, + }; +} + +function createTargetStoreDefault(options: ResolvedMigrationCliOptions): MigrationTargetStore { + const bm25Encoder = createBM25Encoder( + { + enabled: options.bm25Enabled, + language: options.bm25Language, + }, + ); + + return new TcvdbMemoryStore({ + url: options.tcvdb.url, + username: options.tcvdb.username, + apiKey: options.tcvdb.apiKey, + database: options.tcvdb.database, + embeddingModel: options.tcvdb.embeddingModel, + timeout: options.tcvdb.timeout, + caPemPath: options.tcvdb.caPemPath, + bm25Encoder, + }); +} + +async function ensureTargetIsEmpty( + options: ResolvedMigrationCliOptions, + targetStore: MigrationTargetStore, +): Promise { + if (!options.failIfTargetNonempty) return; + + const [existingL1, existingL0, existingProfiles] = await Promise.all([ + Promise.resolve(targetStore.countL1()), + Promise.resolve(targetStore.countL0()), + targetStore.pullProfiles ? targetStore.pullProfiles().then((records) => records.length) : Promise.resolve(0), + ]); + + if (existingL1 > 0 || existingL0 > 0 || existingProfiles > 0) { + throw new Error( + `Target store is not empty (L1=${existingL1}, L0=${existingL0}, profiles=${existingProfiles})`, + ); + } +} + +async function migrateL1Records(sourceStore: VectorStore, targetStore: MigrationTargetStore, pageSize: number): Promise { + let migrated = 0; + let cursor = ""; + const useBatch = typeof targetStore.upsertL1Batch === "function"; + + // eslint-disable-next-line no-constant-condition + while (true) { + const rows = sourceStore.queryL1RecordsCursor(cursor, pageSize); + if (rows.length === 0) break; + + const records = rows.map(mapL1RowToMemoryRecord); + + if (useBatch) { + const count = await targetStore.upsertL1Batch!(records); + if (count === 0) { + throw new Error(`Failed to batch migrate L1 records (cursor=${cursor}, page=${rows.length})`); + } + } else { + for (const record of records) { + const ok = await Promise.resolve(targetStore.upsertL1(record)); + if (!ok) throw new Error(`Failed to migrate L1 record ${record.id}`); + } + } + + migrated += rows.length; + cursor = rows[rows.length - 1].record_id; + log(`L1: 已迁移 ${migrated} 条...`); + + if (rows.length < pageSize) break; + } + + log(`L1: 迁移完成，共 ${migrated} 条`); + return migrated; +} + +async function migrateL0Records(sourceStore: VectorStore, targetStore: MigrationTargetStore, pageSize: number): Promise { + let migrated = 0; + let cursor = ""; + const useBatch = typeof targetStore.upsertL0Batch === "function"; + + // eslint-disable-next-line no-constant-condition + while (true) { + const rows = sourceStore.queryL0RecordsCursor(cursor, pageSize); + if (rows.length === 0) break; + + const records = rows.map(mapL0RowToRecord); + + if (useBatch) { + const count = await targetStore.upsertL0Batch!(records); + if (count === 0) { + throw new Error(`Failed to batch migrate L0 records (cursor=${cursor}, page=${rows.length})`); + } + } else { + for (const record of records) { + const ok = await Promise.resolve(targetStore.upsertL0(record)); + if (!ok) throw new Error(`Failed to migrate L0 record ${record.id}`); + } + } + + migrated += rows.length; + cursor = rows[rows.length - 1].record_id; + log(`L0: 已迁移 ${migrated} 条...`); + + if (rows.length < pageSize) break; + } + + log(`L0: 迁移完成，共 ${migrated} 条`); + return migrated; +} + +async function migrateProfiles( + pluginDataDir: string, + targetStore: MigrationTargetStore, +): Promise { + const profiles = await listLocalProfiles(pluginDataDir); + if (profiles.length === 0) { + log("Profiles: 无本地 profile，跳过"); + return 0; + } + if (!targetStore.syncProfiles) { + throw new Error("Target store does not support profile sync"); + } + log(`Profiles: 发现 ${profiles.length} 个本地 profile，开始同步...`); + await targetStore.syncProfiles(profiles); + log(`Profiles: 同步完成，共 ${profiles.length} 个`); + return profiles.length; +} + +async function verifyMigratedCounts( + summary: MigrationPreflightSummary, + targetStore: MigrationTargetStore, + delayMs: number, +): Promise<{ l1Count: number; l0Count: number; profileCount: number }> { + if (delayMs > 0) { + log(`等待 ${Math.round(delayMs / 1000)} 秒让远端数据落盘...`); + await sleep(delayMs); + } + log("开始校验迁移数量..."); + + const [l1Count, l0Count, profileCount] = await Promise.all([ + Promise.resolve(targetStore.countL1()), + Promise.resolve(targetStore.countL0()), + targetStore.pullProfiles ? targetStore.pullProfiles().then((records) => records.length) : Promise.resolve(0), + ]); + + log(`校验结果: L1=${l1Count}/${summary.source.l1Count}, L0=${l0Count}/${summary.source.l0Count}, Profiles=${profileCount}/${summary.source.profileCount}`); + + if (l1Count !== summary.source.l1Count) { + throw new Error(`L1 count verification failed: source=${summary.source.l1Count}, target=${l1Count}`); + } + if (l0Count !== summary.source.l0Count) { + throw new Error(`L0 count verification failed: source=${summary.source.l0Count}, target=${l0Count}`); + } + if (profileCount !== summary.source.profileCount) { + throw new Error( + `Profile count verification failed: source=${summary.source.profileCount}, target=${profileCount}`, + ); + } + + log("校验通过"); + return { l1Count, l0Count, profileCount }; +} + +async function writeSummaryJson( + summaryJsonPath: string | undefined, + summary: MigrationPreflightSummary, +): Promise { + if (!summaryJsonPath) return; + await fs.mkdir(path.dirname(summaryJsonPath), { recursive: true }); + await fs.writeFile(summaryJsonPath, `${JSON.stringify(summary, null, 2)}\n`, "utf-8"); +} + +function printUsageAndExit(): never { + const text = ` +SQLite → 腾讯云向量数据库迁移工具 + +Usage: + migrate-sqlite-to-tcvdb [options] + +Required: + --plugin-data-dir 插件数据目录路径 + --openclaw-config-path openclaw.json 配置文件路径 + --tcvdb-url TCVDB 服务地址 + --tcvdb-username TCVDB 用户名 + --tcvdb-database TCVDB 数据库名 + --tcvdb-embedding-model Embedding 模型名称 + --tcvdb-api-key TCVDB API 密钥（明文，与 --tcvdb-api-key-env 二选一） + --tcvdb-api-key-env 包含 API 密钥的环境变量名 + +Optional: + --sqlite-path SQLite 数据库路径（默认: /vectors.db） + --plugin-id 写入配置时使用的插件 ID（默认: memory-tencentdb） + --layers 要迁移的层，逗号分隔（默认: l0,l1,l2,l3） + --tcvdb-alias 用户自定义别名 + --tcvdb-timeout-ms 请求超时（默认: 10000） + --tcvdb-ca-pem CA 证书 PEM 文件路径（HTTPS 连接时使用） + --bm25-language BM25 分词语言（默认: zh） + --summary-json-path 将迁移摘要写入此文件 + --job-id 迁移任务 ID（用于追踪） + +Flags: + --dry-run 仅预览，不执行写入 + --yes 跳过交互确认 + --no-apply-config 不自动更新 openclaw.json + --no-config-backup 写入配置前不备份 + --no-rewrite-manifest 不更新 manifest.json + --no-fail-if-target-nonempty 目标库非空时不中止 + --no-verify-counts 迁移后不校验记录数 + --no-bm25-enabled 禁用 BM25 稀疏向量 + + -h, --help 显示此帮助信息 + +Examples: + # 预检模式 + migrate-sqlite-to-tcvdb \\ + --plugin-data-dir ~/.openclaw/memory-tdai \\ + --openclaw-config-path ~/.openclaw/openclaw.json \\ + --tcvdb-url http://127.0.0.1:80 --tcvdb-username root \\ + --tcvdb-api-key-env TCVDB_API_KEY \\ + --tcvdb-database agent_memory_prod \\ + --tcvdb-embedding-model bge-large-zh \\ + --dry-run + + # 正式迁移 + migrate-sqlite-to-tcvdb \\ + --plugin-data-dir ~/.openclaw/memory-tdai \\ + --openclaw-config-path ~/.openclaw/openclaw.json \\ + --tcvdb-url http://127.0.0.1:80 --tcvdb-username root \\ + --tcvdb-api-key-env TCVDB_API_KEY \\ + --tcvdb-database agent_memory_prod \\ + --tcvdb-embedding-model bge-large-zh \\ + --yes +`.trimStart(); + process.stdout.write(text); + process.exit(0); +} + +export function resolveMigrationCliOptions(argv: string[]): ResolvedMigrationCliOptions { + const { values } = parseArgs({ + args: argv, + strict: true, + allowPositionals: false, + allowNegative: true, + options: { + help: { type: "boolean", short: "h" }, + "plugin-data-dir": { type: "string" }, + "sqlite-path": { type: "string" }, + "openclaw-config-path": { type: "string" }, + "plugin-id": { type: "string" }, + layers: { type: "string" }, + "apply-config": { type: "boolean" }, + "config-backup": { type: "boolean" }, + "rewrite-manifest": { type: "boolean" }, + "fail-if-target-nonempty": { type: "boolean" }, + "verify-counts": { type: "boolean" }, + "dry-run": { type: "boolean" }, + yes: { type: "boolean" }, + "bm25-enabled": { type: "boolean" }, + "bm25-language": { type: "string" }, + "summary-json-path": { type: "string" }, + "job-id": { type: "string" }, + "tcvdb-url": { type: "string" }, + "tcvdb-username": { type: "string" }, + "tcvdb-api-key": { type: "string" }, + "tcvdb-api-key-env": { type: "string" }, + "tcvdb-database": { type: "string" }, + "tcvdb-alias": { type: "string" }, + "tcvdb-embedding-model": { type: "string" }, + "tcvdb-timeout-ms": { type: "string" }, + "tcvdb-ca-pem": { type: "string" }, + }, + }); + + if (values.help) { + printUsageAndExit(); + } + + const pluginDataDir = path.resolve(getRequiredString(values, "plugin-data-dir")); + const sqlitePath = path.resolve( + getOptionalString(values, "sqlite-path") ?? path.join(pluginDataDir, "vectors.db"), + ); + const summaryJsonPath = getOptionalString(values, "summary-json-path"); + + return { + pluginDataDir, + sqlitePath, + openclawConfigPath: path.resolve(getRequiredString(values, "openclaw-config-path")), + pluginId: getOptionalString(values, "plugin-id") ?? DEFAULT_MIGRATION_PLUGIN_ID, + layers: parseLayers(getOptionalString(values, "layers")), + applyConfig: resolveBooleanOption(values, "apply-config", true), + configBackup: resolveBooleanOption(values, "config-backup", true), + rewriteManifest: resolveBooleanOption(values, "rewrite-manifest", true), + failIfTargetNonempty: resolveBooleanOption(values, "fail-if-target-nonempty", true), + verifyCounts: resolveBooleanOption(values, "verify-counts", true), + dryRun: resolveBooleanOption(values, "dry-run", false), + yes: resolveBooleanOption(values, "yes", false), + bm25Enabled: resolveBooleanOption(values, "bm25-enabled", true), + bm25Language: parseBm25Language(getOptionalString(values, "bm25-language")), + summaryJsonPath: summaryJsonPath ? path.resolve(summaryJsonPath) : undefined, + jobId: getOptionalString(values, "job-id"), + tcvdb: { + url: getRequiredString(values, "tcvdb-url"), + username: getRequiredString(values, "tcvdb-username"), + apiKey: resolveTcvdbApiKey(values), + database: getRequiredString(values, "tcvdb-database"), + alias: getOptionalString(values, "tcvdb-alias") ?? "", + embeddingModel: getRequiredString(values, "tcvdb-embedding-model"), + timeout: parseTimeout(getOptionalString(values, "tcvdb-timeout-ms")), + caPemPath: getOptionalString(values, "tcvdb-ca-pem"), + }, + }; +} + +export async function collectMigrationPreflight( + options: ResolvedMigrationCliOptions, +): Promise { + // ── 优雅处理"数据目录 / sqlite 不存在"的场景 ── + // 如果源数据路径不存在（全新部署、尚未有任何 capture），不报错—— + // 返回"全零"summary，让调用方判断是否跳过迁移。 + const dirExists = await fs.stat(options.pluginDataDir).then(s => s.isDirectory()).catch(() => false); + if (!dirExists) { + log(`plugin data directory 不存在，无需迁移: ${options.pluginDataDir}`); + return buildEmptySummary(options); + } + const sqliteExists = await fs.access(options.sqlitePath).then(() => true).catch(() => false); + if (!sqliteExists) { + log(`sqlite database 不存在，无需迁移: ${options.sqlitePath}`); + return buildEmptySummary(options); + } + await ensureReadablePath(options.openclawConfigPath, "OpenClaw config file"); + + const store = new VectorStore(options.sqlitePath, 0); + const initResult = store.init(); + if (store.isDegraded()) { + store.close(); + throw new Error(`Failed to open sqlite store for migration preflight: ${initResult.reason ?? "unknown error"}`); + } + + try { + const [profiles, manifest] = await Promise.all([ + listLocalProfiles(options.pluginDataDir), + Promise.resolve(readManifest(options.pluginDataDir)), + ]); + + return { + pluginId: options.pluginId, + dryRun: options.dryRun, + layers: options.layers, + paths: { + pluginDataDir: options.pluginDataDir, + sqlitePath: options.sqlitePath, + openclawConfigPath: options.openclawConfigPath, + }, + source: { + l0Count: store.countL0(), + l1Count: store.countL1(), + profileCount: profiles.length, + manifestExists: manifest !== null, + manifestStoreType: manifest?.store.type ?? null, + }, + target: { + url: options.tcvdb.url, + username: options.tcvdb.username, + database: options.tcvdb.database, + alias: options.tcvdb.alias, + embeddingModel: options.tcvdb.embeddingModel, + timeout: options.tcvdb.timeout, + bm25Enabled: options.bm25Enabled, + bm25Language: options.bm25Language, + }, + options: { + applyConfig: options.applyConfig, + configBackup: options.configBackup, + rewriteManifest: options.rewriteManifest, + failIfTargetNonempty: options.failIfTargetNonempty, + verifyCounts: options.verifyCounts, + yes: options.yes, + }, + }; + } finally { + store.close(); + } +} + +export async function runMigrationCli( + argv: string[], + deps: RunMigrationCliDeps = {}, +): Promise { + const options = resolveMigrationCliOptions(argv); + log("开始预检..."); + const summary = await collectMigrationPreflight(options); + log(`预检完成: 源数据 L1=${summary.source.l1Count}, L0=${summary.source.l0Count}, Profiles=${summary.source.profileCount}`); + log(`目标: ${summary.target.url} / ${summary.target.database}`); + + const hasSourceData = summary.source.l0Count > 0 || summary.source.l1Count > 0 || summary.source.profileCount > 0; + + if (!hasSourceData) { + log("源数据为空，跳过数据迁移。"); + } + + if (options.dryRun) { + log("预检模式 (dry-run)，不执行写入"); + await writeSummaryJson(options.summaryJsonPath, summary); + return summary; + } + + const createTargetStore = deps.createTargetStore ?? createTargetStoreDefault; + const writeMigrationPluginConfig = deps.writeMigrationPluginConfig ?? writeMigrationPluginConfigDefault; + const rewriteMigrationManifest = deps.rewriteMigrationManifest ?? (async (params) => + await rewriteMigrationManifestDefault(params)); + + const migration = { + l1Migrated: 0, + l0Migrated: 0, + profileMigrated: 0, + targetL1Count: 0, + targetL0Count: 0, + targetProfileCount: 0, + configWritten: false, + manifestWritten: false, + manifestBackupPath: undefined as string | undefined, + }; + + if (hasSourceData) { + log("打开 SQLite 源库..."); + const sourceStore = new VectorStore(options.sqlitePath, 0); + const sourceInitResult = sourceStore.init(); + if (sourceStore.isDegraded()) { + sourceStore.close(); + throw new Error(`Failed to reopen sqlite source store: ${sourceInitResult.reason ?? "unknown error"}`); + } + + log("初始化目标库..."); + const targetStore = createTargetStore(options); + + try { + await Promise.resolve(targetStore.init()); + if (targetStore.isDegraded()) { + throw new Error("Target store entered degraded mode during initialization"); + } + + await ensureTargetIsEmpty(options, targetStore); + + const pageSize = DEFAULT_MIGRATION_PAGE_SIZE; + log(`分页大小: ${pageSize} 条/批`); + + migration.l1Migrated = options.layers.includes("l1") ? await migrateL1Records(sourceStore, targetStore, pageSize) : 0; + migration.l0Migrated = options.layers.includes("l0") ? await migrateL0Records(sourceStore, targetStore, pageSize) : 0; + migration.profileMigrated = options.layers.includes("l2") || options.layers.includes("l3") + ? await migrateProfiles(options.pluginDataDir, targetStore) + : 0; + + const verifyDelayMs = deps.verifyDelayMs ?? 10_000; + + const verifiedCounts = options.verifyCounts + ? await verifyMigratedCounts(summary, targetStore, verifyDelayMs) + : { + l1Count: await Promise.resolve(targetStore.countL1()), + l0Count: await Promise.resolve(targetStore.countL0()), + profileCount: targetStore.pullProfiles ? (await targetStore.pullProfiles()).length : 0, + }; + + migration.targetL1Count = verifiedCounts.l1Count; + migration.targetL0Count = verifiedCounts.l0Count; + migration.targetProfileCount = verifiedCounts.profileCount; + } finally { + sourceStore.close(); + targetStore.close(); + } + } + + if (options.applyConfig) { + log(`写入配置到 ${options.openclawConfigPath} ...`); + await writeMigrationPluginConfig({ + configPath: options.openclawConfigPath, + pluginId: options.pluginId, + tcvdb: { + url: options.tcvdb.url, + username: options.tcvdb.username, + apiKey: options.tcvdb.apiKey, + database: options.tcvdb.database, + alias: options.tcvdb.alias, + embeddingModel: options.tcvdb.embeddingModel, + timeout: options.tcvdb.timeout, + }, + bm25: { + enabled: options.bm25Enabled, + language: options.bm25Language, + }, + }); + migration.configWritten = true; + log("配置写入完成"); + } + + if (options.rewriteManifest) { + log("更新 manifest..."); + const manifestResult = await rewriteMigrationManifest({ + dataDir: options.pluginDataDir, + tcvdbUrl: options.tcvdb.url, + tcvdbDatabase: options.tcvdb.database, + tcvdbAlias: options.tcvdb.alias || undefined, + }); + migration.manifestWritten = manifestResult.created || manifestResult.updated; + migration.manifestBackupPath = manifestResult.backupPath; + log(`Manifest ${manifestResult.created ? "创建" : "更新"}完成${manifestResult.backupPath ? `，备份: ${manifestResult.backupPath}` : ""}`); + } + + summary.migration = { + l1Migrated: migration.l1Migrated, + l0Migrated: migration.l0Migrated, + profileMigrated: migration.profileMigrated, + targetL1Count: migration.targetL1Count, + targetL0Count: migration.targetL0Count, + targetProfileCount: migration.targetProfileCount, + configWritten: migration.configWritten, + manifestWritten: migration.manifestWritten, + manifestBackupPath: migration.manifestBackupPath, + }; + + await writeSummaryJson(options.summaryJsonPath, summary); + log("迁移全部完成!"); + return summary; +} diff --git a/scripts/migrate-sqlite-to-tcvdb/tsconfig.json b/scripts/migrate-sqlite-to-tcvdb/tsconfig.json new file mode 100644 index 0000000..dac3f3b --- /dev/null +++ b/scripts/migrate-sqlite-to-tcvdb/tsconfig.json @@ -0,0 +1,25 @@ +{ + "compilerOptions": { + "target": "es2023", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "declaration": false, + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "resolveJsonModule": true, + "types": ["node"], + "paths": { + "node-llama-cpp": ["./node-llama-cpp.d.ts"] + }, + "outDir": "dist", + "rootDir": "../.." + }, + "files": [ + "cli-entry.ts", + "sqlite-to-tcvdb.ts", + "config-write.ts", + "manifest-write.ts" + ] +} diff --git a/scripts/read-local-memory/read-local-memory.ts b/scripts/read-local-memory/read-local-memory.ts new file mode 100644 index 0000000..952977c --- /dev/null +++ b/scripts/read-local-memory/read-local-memory.ts @@ -0,0 +1,1057 @@ +#!/usr/bin/env npx tsx +/** + * 本地 Memory 数据查询脚本 + * + * 查询 memory-tdai 目录下的记忆数据，支持： + * - 按层级（L0~L3）查询 + * - L0/L1 从 SQLite（vectors.db）读取 + * - 时间范围过滤（--since / --until） + * - 字段过滤（--filter，仅支持 SQLite 表的直接列） + * - 排序、分页（下推到 SQL 层） + * - 多种输出格式（table / json / jsonl） + * + * @example + * npx tsx read-local-memory.ts -d ./memory-tdai示例数据 + * npx tsx read-local-memory.ts -d ./memory-tdai示例数据 -L L0 --since 7d + * npx tsx read-local-memory.ts -d ./memory-tdai示例数据 -L L1 -f 'type=persona' + */ + +import { createRequire } from "node:module" +import type { DatabaseSync } from "node:sqlite" +import * as fs from "node:fs" +import * as path from "node:path" +import { parseArgs } from "node:util" + +const require = createRequire(import.meta.url) + +function requireNodeSqlite(): typeof import("node:sqlite") { + return require("node:sqlite") as typeof import("node:sqlite") +} + +// ───────────────────────────────────────────── +// Types +// ───────────────────────────────────────────── + +type Level = "L0" | "L1" | "L2" | "L3" +type SortDirection = "asc" | "desc" +type OutputFormat = "table" | "json" | "jsonl" + +interface CliOptions { + dataDir: string + level?: Level + since?: string + until?: string + limit: number + offset: number + sort: SortDirection + filter?: string + format: OutputFormat + file?: string // L2 单文件详情查询：指定文件名，只返回该文件的完整内容 +} + +interface FilterCondition { + field: string + operator: "=" | "!=" | ">=" | "<=" | ">" | "<" + value: string +} + +interface L2Meta { + created: string + updated: string + summary: string + heat: number + [key: string]: string | number +} + +interface L2Entry { + fileName: string + meta: L2Meta + body: string +} + +interface QueryResult { + level: string + total: number + offset: number + limit: number + sort: SortDirection + filter: Record | null + data: T[] +} + +// ───────────────────────────────────────────── +// Constants +// ───────────────────────────────────────────── + +const SQLITE_DB_NAME = "vectors.db" + +const LEVEL_DIRS: Record = { + L2: "scene_blocks", + L3: "persona.md", +} + +/** L0 表的允许过滤列（白名单防 SQL 注入） */ +const L0_FILTER_COLUMNS = new Set([ + "record_id", "session_key", "session_id", "role", "message_text", "recorded_at", "timestamp", +]) + +/** L1 表的允许过滤列（白名单防 SQL 注入） */ +const L1_FILTER_COLUMNS = new Set([ + "record_id", "content", "type", "priority", "scene_name", + "session_key", "session_id", "timestamp_str", "timestamp_start", "timestamp_end", + "created_time", "updated_time", "metadata_json", +]) + +/** 驼峰字段名 → SQLite 列名映射（用户用驼峰过滤，内部转成 SQL 列名） */ +const CAMEL_TO_COLUMN: Record = { + id: "record_id", + recordId: "record_id", + sessionKey: "session_key", + sessionId: "session_id", + messageText: "message_text", + recordedAt: "recorded_at", + sceneName: "scene_name", + timestampStr: "timestamp_str", + timestampStart: "timestamp_start", + timestampEnd: "timestamp_end", + createdAt: "created_time", + updatedAt: "updated_time", + metadataJson: "metadata_json", +} + +const META_START = "-----META-START-----" +const META_END = "-----META-END-----" + +const RELATIVE_TIME_RE = /^(\d+)(d|h|m|s)$/ + +const HELP_TEXT = ` +📖 本地 Memory 数据查询脚本（SQLite 模式） + +Usage: + npx tsx read-local-memory.ts -d <数据目录> [选项] + +数据目录下须包含 vectors.db（SQLite 数据库），L0/L1 数据从中读取。 + +Options: + -d, --data-dir <路径> 本地 memory-tdai 数据目录路径（必填，须含 vectors.db） + -L, --level <层级> 查询层级: L0 / L1 / L2 / L3（不指定则查所有） + --since <时间> 起始时间（ISO 字符串或相对表达式如 7d, 24h, 30m） + --until <时间> 截止时间（同 since 格式） + -l, --limit <数量> 每页返回数量（默认 50） + --offset <偏移> 分页偏移（默认 0） + --sort <方向> 排序: desc（新→旧）/ asc（旧→新），默认 desc + -f, --filter <表达式> 字段过滤，仅支持表的直接列（如 role=user, type=persona, priority>=80） + 支持驼峰或蛇形列名，多条件用逗号分隔 + --format <格式> 输出: table / json / jsonl（默认 table） + -h, --help 显示帮助 + +L0 可过滤列: record_id, session_key, session_id, role, message_text, recorded_at, timestamp +L1 可过滤列: record_id, content, type, priority, scene_name, session_key, session_id, + timestamp_str, timestamp_start, timestamp_end, created_time, updated_time + +Examples: + # 查看所有层级概览 + npx tsx read-local-memory.ts -d ./memory-tdai示例数据 + + # 查询 L0 近 7 天的对话 + npx tsx read-local-memory.ts -d ./memory-tdai示例数据 -L L0 --since 7d + + # 查询 L1 记忆，只看 persona 类型 + npx tsx read-local-memory.ts -d ./memory-tdai示例数据 -L L1 -f 'type=persona' + + # L0 分页：第 2 页（每页 20 条） + npx tsx read-local-memory.ts -d ./memory-tdai示例数据 -L L0 -l 20 --offset 20 + + # 以 JSON 格式输出 + npx tsx read-local-memory.ts -d ./memory-tdai示例数据 -L L0 --since 7d --format json +`.trim() + +// ───────────────────────────────────────────── +// CLI Argument Parsing +// ───────────────────────────────────────────── + +function parseCli(): CliOptions { + const { values } = parseArgs({ + options: { + "data-dir": { type: "string", short: "d" }, + level: { type: "string", short: "L" }, + since: { type: "string" }, + until: { type: "string" }, + limit: { type: "string", short: "l" }, + offset: { type: "string" }, + sort: { type: "string" }, + filter: { type: "string", short: "f" }, + format: { type: "string" }, + file: { type: "string" }, + help: { type: "boolean", short: "h" }, + }, + strict: true, + allowPositionals: false, + }) + + if (values.help) { + console.log(HELP_TEXT) + process.exit(0) + } + + const dataDir = values["data-dir"] + if (!dataDir) { + console.error("❌ 缺少必填参数: --data-dir (-d)") + console.error(' 使用 --help 查看用法') + process.exit(1) + } + + const resolvedDir = path.resolve(dataDir) + if (!fs.existsSync(resolvedDir)) { + console.error(`❌ 数据目录不存在: ${resolvedDir}`) + process.exit(1) + } + + const level = values.level?.toUpperCase() as Level | undefined + if (level && !["L0", "L1", "L2", "L3"].includes(level)) { + console.error(`❌ 无效的层级: ${values.level} （可选: L0, L1, L2, L3）`) + process.exit(1) + } + + const sort = (values.sort?.toLowerCase() ?? "desc") as SortDirection + if (!["asc", "desc"].includes(sort)) { + console.error(`❌ 无效的排序方向: ${values.sort} （可选: asc, desc）`) + process.exit(1) + } + + const format = (values.format?.toLowerCase() ?? "table") as OutputFormat + if (!["table", "json", "jsonl"].includes(format)) { + console.error(`❌ 无效的输出格式: ${values.format} （可选: table, json, jsonl）`) + process.exit(1) + } + + const limit = values.limit ? parseInt(values.limit, 10) : 50 + const offset = values.offset ? parseInt(values.offset, 10) : 0 + + if (isNaN(limit) || limit < 1) { + console.error(`❌ 无效的 limit: ${values.limit}`) + process.exit(1) + } + if (isNaN(offset) || offset < 0) { + console.error(`❌ 无效的 offset: ${values.offset}`) + process.exit(1) + } + + return { + dataDir: resolvedDir, + level, + since: values.since, + until: values.until, + limit, + offset, + sort, + filter: values.filter, + format, + file: values.file, + } +} + +// ───────────────────────────────────────────── +// Time Parsing +// ───────────────────────────────────────────── + +/** 将时间表达式解析为 Date 对象。支持 ISO 字符串或相对表达式（7d / 24h / 30m / 60s） */ +function parseTimeExpr(expr: string): Date { + const match = expr.match(RELATIVE_TIME_RE) + if (match) { + const [, numStr, unit] = match + const num = parseInt(numStr, 10) + const now = Date.now() + const ms: Record = { + d: 86_400_000, + h: 3_600_000, + m: 60_000, + s: 1_000, + } + return new Date(now - num * ms[unit]) + } + + const date = new Date(expr) + if (isNaN(date.getTime())) { + console.error(`❌ 无法解析时间: ${expr}`) + process.exit(1) + } + return date +} + +/** 将 L0 的 epoch ms 或 L1 的 ISO 字符串统一转换为 Date */ +function toDate(value: unknown): Date | null { + if (typeof value === "number") return new Date(value) + if (typeof value === "string") { + const d = new Date(value) + return isNaN(d.getTime()) ? null : d + } + return null +} + +// ───────────────────────────────────────────── +// Filter Parsing +// ───────────────────────────────────────────── + +const FILTER_OPERATORS = [">=", "<=", "!=", ">", "<", "="] as const + +/** SQL 操作符映射（!= → <> for SQLite） */ +const SQL_OPERATOR_MAP: Record = { + "=": "=", + "!=": "<>", + ">=": ">=", + "<=": "<=", + ">": ">", + "<": "<", +} + +function parseFilterExpr(expr: string): FilterCondition[] { + return expr.split(",").map((part) => { + const trimmed = part.trim() + for (const op of FILTER_OPERATORS) { + const idx = trimmed.indexOf(op) + if (idx > 0) { + return { + field: trimmed.slice(0, idx).trim(), + operator: op as FilterCondition["operator"], + value: trimmed.slice(idx + op.length).trim(), + } + } + } + console.error(`❌ 无法解析过滤条件: ${trimmed}`) + process.exit(1) + }) +} + +/** 将用户传入的字段名解析为 SQLite 列名（支持驼峰和蛇形） */ +function resolveColumnName(field: string, allowedColumns: Set): string { + // 直接匹配蛇形列名 + if (allowedColumns.has(field)) return field + // 尝试驼峰转换 + const mapped = CAMEL_TO_COLUMN[field] + if (mapped && allowedColumns.has(mapped)) return mapped + return field // 返回原值，后续校验会报错 +} + +/** 校验过滤条件的列名是否在白名单中 */ +function validateFilterColumns(conditions: FilterCondition[], allowedColumns: Set, level: string): void { + for (const c of conditions) { + const col = resolveColumnName(c.field, allowedColumns) + if (!allowedColumns.has(col)) { + console.error(`❌ ${level} 不支持的过滤字段: ${c.field}`) + console.error(` 可用字段: ${[...allowedColumns].join(", ")}`) + process.exit(1) + } + } +} + +function filtersToRecord(conditions: FilterCondition[]): Record { + const result: Record = {} + for (const c of conditions) { + result[c.field] = `${c.operator}${c.value}` + } + return result +} + +function filtersToDisplayString(conditions: FilterCondition[]): string { + return conditions.map((c) => `${c.field}${c.operator}${c.value}`).join(", ") +} + +// ───────────────────────────────────────────── +// SQLite Helpers +// ───────────────────────────────────────────── + +/** 只读打开 SQLite 数据库 */ +function openSqliteReadonly(dbPath: string): DatabaseSync { + const { DatabaseSync: DbSync } = requireNodeSqlite() + const db = new DbSync(dbPath, { open: false }) + // node:sqlite 没有直接的 readOnly 选项，用 query_only pragma 保证只读 + db.open() + db.exec("PRAGMA query_only = ON") + return db +} + +interface SqlQueryResult { + total: number + records: Record[] +} + +/** + * 构建 WHERE 子句（时间过滤 + 字段过滤），返回 SQL 片段和参数。 + * 所有过滤条件通过参数化查询绑定，防止 SQL 注入。 + */ +function buildWhereClause( + level: "L0" | "L1", + sinceDate: Date | null, + untilDate: Date | null, + filterConditions: FilterCondition[] | null, +): { whereClause: string; params: (string | number)[] } { + const clauses: string[] = [] + const params: (string | number)[] = [] + const allowedColumns = level === "L0" ? L0_FILTER_COLUMNS : L1_FILTER_COLUMNS + + // 时间过滤 + if (level === "L0") { + // L0: timestamp 是 epoch ms (INTEGER) + if (sinceDate) { + clauses.push("timestamp >= ?") + params.push(sinceDate.getTime()) + } + if (untilDate) { + clauses.push("timestamp <= ?") + params.push(untilDate.getTime()) + } + } else { + // L1: updated_time 是 ISO 字符串 (TEXT) + if (sinceDate) { + clauses.push("updated_time >= ?") + params.push(sinceDate.toISOString()) + } + if (untilDate) { + clauses.push("updated_time <= ?") + params.push(untilDate.toISOString()) + } + } + + // 字段过滤 + if (filterConditions) { + for (const c of filterConditions) { + const col = resolveColumnName(c.field, allowedColumns) + const sqlOp = SQL_OPERATOR_MAP[c.operator] + clauses.push(`${col} ${sqlOp} ?`) + // 如果值可解析为数字且列是数字类型，传数字；否则传字符串 + const numVal = Number(c.value) + params.push(!isNaN(numVal) && c.value.trim() !== "" ? numVal : c.value) + } + } + + const whereClause = clauses.length > 0 ? `WHERE ${clauses.join(" AND ")}` : "" + return { whereClause, params } +} + +/** L0 SQLite 行 → 驼峰命名输出对象 */ +function mapL0Row(row: Record): Record { + return { + id: row.record_id, + sessionKey: row.session_key, + sessionId: row.session_id, + role: row.role, + content: row.message_text, + recordedAt: row.recorded_at, + timestamp: row.timestamp, + } +} + +/** L1 SQLite 行 → 驼峰命名输出对象 */ +function mapL1Row(row: Record): Record { + const metadataRaw = row.metadata_json as string + let metadata: unknown = {} + try { + metadata = metadataRaw ? JSON.parse(metadataRaw) : {} + } catch { + metadata = {} + } + + const timestamps = [ + ...(new Set( + [row.timestamp_str, row.timestamp_start, row.timestamp_end] + .filter(Boolean) as string[] + )) + ] + + return { + id: row.record_id, + content: row.content, + type: row.type, + priority: row.priority, + scene_name: row.scene_name, + source_message_ids: [], + metadata, + timestamps, + createdAt: row.created_time || "", + updatedAt: row.updated_time || "", + sessionKey: row.session_key || "", + sessionId: row.session_id || "", + } +} + +function querySqlite(db: DatabaseSync, level: "L0" | "L1", opts: CliOptions): SqlQueryResult { + const table = level === "L0" ? "l0_conversations" : "l1_records" + const timeCol = level === "L0" ? "timestamp" : "updated_time" + const allowedColumns = level === "L0" ? L0_FILTER_COLUMNS : L1_FILTER_COLUMNS + + const sinceDate = opts.since ? parseTimeExpr(opts.since) : null + const untilDate = opts.until ? parseTimeExpr(opts.until) : null + + let filterConditions: FilterCondition[] | null = null + if (opts.filter) { + filterConditions = parseFilterExpr(opts.filter) + validateFilterColumns(filterConditions, allowedColumns, level) + } + + const { whereClause, params } = buildWhereClause(level, sinceDate, untilDate, filterConditions) + + // 查总数 + const countSql = `SELECT COUNT(*) AS cnt FROM ${table} ${whereClause}` + const countRow = db.prepare(countSql).get(...params) as { cnt: number } + const total = countRow.cnt + + // 查数据（排序 + 分页） + const sortDir = opts.sort === "asc" ? "ASC" : "DESC" + const dataSql = `SELECT * FROM ${table} ${whereClause} ORDER BY ${timeCol} ${sortDir} LIMIT ? OFFSET ?` + const dataParams: (string | number)[] = [...params, opts.limit, opts.offset] + const rows = db.prepare(dataSql).all(...dataParams) as Record[] + + // 映射为驼峰命名 + const mapFn = level === "L0" ? mapL0Row : mapL1Row + const records = rows.map(mapFn) + + return { total, records } +} + +// ───────────────────────────────────────────── +// Query: L0 / L1 (SQLite) +// ───────────────────────────────────────────── + +function querySqliteLevel(db: DatabaseSync, opts: CliOptions, level: "L0" | "L1") { + const { total, records: paged } = querySqlite(db, level, opts) + + const timeField = level === "L0" ? "timestamp" : "updatedAt" + const levelLabel = level === "L0" ? "conversations" : "records" + + let filterConditions: FilterCondition[] | null = null + if (opts.filter) { + filterConditions = parseFilterExpr(opts.filter) + } + const filterRecord = filterConditions ? filtersToRecord(filterConditions) : null + const filterDisplay = filterConditions ? filtersToDisplayString(filterConditions) : "" + const sinceInfo = opts.since ? `since=${opts.since}` : "" + const untilInfo = opts.until ? `until=${opts.until}` : "" + const filterParts = [filterDisplay, sinceInfo, untilInfo].filter(Boolean) + + if (opts.format === "json") { + const result: QueryResult> = { + level, + total, + offset: opts.offset, + limit: opts.limit, + sort: opts.sort, + filter: filterRecord, + data: paged, + } + console.log(JSON.stringify(result)) + return + } + + if (opts.format === "jsonl") { + for (const record of paged) { + console.log(JSON.stringify(record)) + } + return + } + + // ── table 格式 ── + const rangeStart = total === 0 ? 0 : opts.offset + 1 + const rangeEnd = Math.min(opts.offset + opts.limit, total) + + console.log() + console.log(`📊 查询结果：${level} ${levelLabel}（SQLite）`) + console.log(` 总条数: ${total}`) + console.log(` 当前页: ${rangeStart}-${rangeEnd} / ${total}（按 ${timeField} ${opts.sort === "desc" ? "降序" : "升序"}）`) + if (filterParts.length > 0) { + console.log(` 过滤条件: ${filterParts.join(", ")}`) + } + console.log() + + if (paged.length === 0) { + console.log(" （无匹配数据）") + console.log() + return + } + + if (level === "L0") { + renderL0Table(paged) + } else { + renderL1Table(paged) + } +} + +/** 截断字符串并添加省略号 */ +function truncate(str: string, maxLen: number): string { + if (!str) return "" + const clean = str.replace(/\n/g, "↵").replace(/\r/g, "") + if (clean.length <= maxLen) return clean + return clean.slice(0, maxLen - 1) + "…" +} + +/** 计算字符串的显示宽度（CJK 字符占 2 宽） */ +function displayWidth(str: string): number { + let width = 0 + for (const char of str) { + const code = char.codePointAt(0)! + // CJK Unified Ideographs / fullwidth / common CJK ranges + if ( + (code >= 0x4e00 && code <= 0x9fff) || // CJK 基本 + (code >= 0x3000 && code <= 0x303f) || // CJK 标点 + (code >= 0xff00 && code <= 0xffef) || // 全角 + (code >= 0x3400 && code <= 0x4dbf) || // CJK 扩展A + (code >= 0x20000 && code <= 0x2a6df) || // CJK 扩展B + (code >= 0xf900 && code <= 0xfaff) // CJK 兼容 + ) { + width += 2 + } else { + width += 1 + } + } + return width +} + +/** 将字符串右填充到指定显示宽度 */ +function padEnd(str: string, targetWidth: number): string { + const diff = targetWidth - displayWidth(str) + return diff > 0 ? str + " ".repeat(diff) : str +} + +/** 将字符串居中到指定显示宽度 */ +function padCenter(str: string, targetWidth: number): string { + const diff = targetWidth - displayWidth(str) + if (diff <= 0) return str + const left = Math.floor(diff / 2) + const right = diff - left + return " ".repeat(left) + str + " ".repeat(right) +} + +/** 打印表格 */ +function printTable(headers: string[], rows: string[][], colWidths: number[]) { + const hLine = (left: string, mid: string, right: string, fill: string) => + left + colWidths.map((w) => fill.repeat(w + 2)).join(mid) + right + + console.log(hLine("┌", "┬", "┐", "─")) + + const headerRow = headers.map((h, i) => ` ${padCenter(h, colWidths[i])} `).join("│") + console.log(`│${headerRow}│`) + + console.log(hLine("├", "┼", "┤", "─")) + + for (const row of rows) { + const line = row.map((cell, i) => ` ${padEnd(cell, colWidths[i])} `).join("│") + console.log(`│${line}│`) + } + + console.log(hLine("└", "┴", "┘", "─")) +} + +/** 格式化时间为可读字符串 */ +function formatTime(value: unknown): string { + const date = toDate(value) + if (!date) return String(value ?? "") + const y = date.getFullYear() + const M = String(date.getMonth() + 1).padStart(2, "0") + const d = String(date.getDate()).padStart(2, "0") + const h = String(date.getHours()).padStart(2, "0") + const m = String(date.getMinutes()).padStart(2, "0") + return `${y}-${M}-${d} ${h}:${m}` +} + +// ───────────────────────────────────────────── +// File I/O Helpers (L2 Markdown) +// ───────────────────────────────────────────── + +/** 读取并解析 L2 Markdown 文件（含 META 头） */ +function parseL2File(filePath: string): L2Entry { + const content = fs.readFileSync(filePath, "utf-8") + const fileName = path.basename(filePath) + + const startIdx = content.indexOf(META_START) + const endIdx = content.indexOf(META_END) + + const meta: L2Meta = { created: "", updated: "", summary: "", heat: 0 } + let body = content + + if (startIdx !== -1 && endIdx !== -1) { + const metaBlock = content.slice(startIdx + META_START.length, endIdx).trim() + + for (const line of metaBlock.split("\n")) { + const colonIdx = line.indexOf(":") + if (colonIdx > 0) { + const key = line.slice(0, colonIdx).trim() + const val = line.slice(colonIdx + 1).trim() + if (key === "heat") { + meta.heat = parseInt(val, 10) || 0 + } else { + ;(meta as Record)[key] = val + } + } + } + + body = content.slice(endIdx + META_END.length).trim() + } + + return { fileName, meta, body } +} + +function renderL0Table(records: Record[]) { + const headers = ["#", "timestamp", "role", "content"] + const colWidths = [5, 18, 10, 50] + + const rows = records.map((r, i) => [ + String(i + 1), + formatTime(r.timestamp), + truncate(String(r.role ?? ""), 10), + truncate(String(r.content ?? ""), 50), + ]) + + // 动态调整内容列宽（至少 30，至多 80） + const maxContentWidth = Math.min( + 80, + Math.max(30, ...rows.map((r) => displayWidth(r[3]))) + ) + colWidths[3] = maxContentWidth + + printTable(headers, rows, colWidths) + console.log() +} + +function renderL1Table(records: Record[]) { + const headers = ["#", "updatedAt", "type", "pri", "content"] + const colWidths = [5, 18, 12, 4, 50] + + const rows = records.map((r, i) => [ + String(i + 1), + formatTime(r.updatedAt), + truncate(String(r.type ?? ""), 12), + String(r.priority ?? ""), + truncate(String(r.content ?? ""), 50), + ]) + + const maxContentWidth = Math.min( + 80, + Math.max(30, ...rows.map((r) => displayWidth(r[4]))) + ) + colWidths[4] = maxContentWidth + + printTable(headers, rows, colWidths) + console.log() +} + +// ───────────────────────────────────────────── +// Query: L2 (Scene Blocks) +// ───────────────────────────────────────────── + +function queryL2(opts: CliOptions) { + const dirPath = path.join(opts.dataDir, LEVEL_DIRS.L2) + + if (!fs.existsSync(dirPath)) { + // 目录不存在是正常业务场景（尚未产生场景数据），返回空数据 + if (opts.format === "json") { + console.log(JSON.stringify({ level: "L2", total: 0, data: [] })) + return + } + if (opts.format === "jsonl") { + return + } + console.log() + console.log(`📊 查询结果：L2 scene_blocks`) + console.log(` （尚未生成场景数据）`) + console.log() + return + } + + const files = fs.readdirSync(dirPath).filter((f) => f.endsWith(".md")).sort() + const entries: L2Entry[] = files.map((f) => parseL2File(path.join(dirPath, f))) + + // --file 参数：只返回指定文件的完整内容（含 body） + if (opts.file) { + const target = entries.find((e) => e.fileName === opts.file) + if (!target) { + console.error(`❌ 文件不存在: ${opts.file}`) + process.exit(1) + } + if (opts.format === "json") { + console.log(JSON.stringify({ + level: "L2", + fileName: target.fileName, + ...target.meta, + body: target.body, + })) + return + } + // table / jsonl 格式直接输出文件内容 + console.log(target.body) + return + } + + if (opts.format === "json") { + // 默认列表模式：只输出元信息（不含 body），避免超过 TAT 24KB 输出限制 + const result = { + level: "L2", + total: entries.length, + data: entries.map(({ fileName, meta }) => ({ + fileName, + ...meta, + })), + } + console.log(JSON.stringify(result)) + return + } + + if (opts.format === "jsonl") { + for (const { fileName, meta, body } of entries) { + console.log(JSON.stringify({ fileName, ...meta, body })) + } + return + } + + // ── table 格式 ── + console.log() + console.log(`📊 查询结果：L2 scene_blocks`) + console.log(` 总文件数: ${entries.length}`) + console.log() + + if (entries.length === 0) { + console.log(" （无场景画像文件）") + console.log() + return + } + + for (const { fileName, meta, body } of entries) { + console.log(`${"─".repeat(60)}`) + console.log(`📄 ${fileName}`) + console.log(` Summary : ${meta.summary}`) + console.log(` Heat : ${meta.heat}`) + console.log(` Created : ${meta.created}`) + console.log(` Updated : ${meta.updated}`) + console.log() + + // 输出正文（限制行数避免过长） + const lines = body.split("\n") + const maxLines = 30 + if (lines.length > maxLines) { + console.log(lines.slice(0, maxLines).join("\n")) + console.log(` ... (省略 ${lines.length - maxLines} 行，共 ${lines.length} 行)`) + } else { + console.log(body) + } + console.log() + } +} + +// ───────────────────────────────────────────── +// Query: L3 (Persona) +// ───────────────────────────────────────────── + +function queryL3(opts: CliOptions) { + const filePath = path.join(opts.dataDir, LEVEL_DIRS.L3) + + // 文件不存在是正常业务场景（用户还没对话、插件刚安装等），返回空数据 + if (!fs.existsSync(filePath)) { + if (opts.format === "json") { + console.log(JSON.stringify({ level: "L3", content: "" })) + return + } + if (opts.format === "jsonl") { + console.log(JSON.stringify({ level: "L3", content: "" })) + return + } + console.log() + console.log(`📊 查询结果：L3 persona`) + console.log(` （画像文件尚未生成）`) + console.log() + return + } + + const content = fs.readFileSync(filePath, "utf-8") + + if (opts.format === "json") { + console.log(JSON.stringify({ level: "L3", content })) + return + } + + if (opts.format === "jsonl") { + console.log(JSON.stringify({ level: "L3", content })) + return + } + + console.log() + console.log(`📊 查询结果：L3 persona`) + console.log(`${"─".repeat(60)}`) + console.log(content) + console.log() +} + +// ───────────────────────────────────────────── +// Overview: 全层级概览 +// ───────────────────────────────────────────── + +function showOverview(db: DatabaseSync, opts: CliOptions) { + console.log() + console.log(`🗂️ Memory 数据概览`) + console.log(` 数据目录: ${opts.dataDir}`) + console.log(` 数据库: ${SQLITE_DB_NAME}`) + console.log(`${"═".repeat(60)}`) + + // ── L0 ── + try { + const l0Count = (db.prepare("SELECT COUNT(*) AS cnt FROM l0_conversations").get() as { cnt: number }).cnt + const l0Roles = db.prepare("SELECT role, COUNT(*) AS cnt FROM l0_conversations GROUP BY role").all() as Array<{ role: string; cnt: number }> + const roleSummary = l0Roles.map((r) => `${r.role || "unknown"}: ${r.cnt}`).join(", ") + + console.log() + console.log(`📂 L0 · conversations (l0_conversations)`) + console.log(` 总条数: ${l0Count}`) + if (roleSummary) { + console.log(` 角色分布: ${roleSummary}`) + } + } catch { + console.log() + console.log(`📂 L0 · conversations （表不存在或查询失败）`) + } + + // ── L1 ── + try { + const l1Count = (db.prepare("SELECT COUNT(*) AS cnt FROM l1_records").get() as { cnt: number }).cnt + const l1Types = db.prepare("SELECT type, COUNT(*) AS cnt FROM l1_records GROUP BY type").all() as Array<{ type: string; cnt: number }> + const typeSummary = l1Types.map((t) => `${t.type || "unknown"}: ${t.cnt}`).join(", ") + + console.log() + console.log(`📂 L1 · records (l1_records)`) + console.log(` 总条数: ${l1Count}`) + if (typeSummary) { + console.log(` 类型分布: ${typeSummary}`) + } + } catch { + console.log() + console.log(`📂 L1 · records （表不存在或查询失败）`) + } + + // ── L2 ── + const l2Dir = path.join(opts.dataDir, LEVEL_DIRS.L2) + if (fs.existsSync(l2Dir)) { + const files = fs.readdirSync(l2Dir).filter((f) => f.endsWith(".md")) + const entries = files.map((f) => parseL2File(path.join(l2Dir, f))) + const totalHeat = entries.reduce((sum, e) => sum + e.meta.heat, 0) + + console.log() + console.log(`📂 L2 · scene_blocks`) + console.log(` 文件数: ${files.length} 总热度: ${totalHeat}`) + for (const entry of entries) { + console.log(` · ${entry.fileName} (heat: ${entry.meta.heat}) ${truncate(entry.meta.summary, 40)}`) + } + } else { + console.log() + console.log(`📂 L2 · scene_blocks （目录不存在）`) + } + + // ── L3 ── + const l3Path = path.join(opts.dataDir, LEVEL_DIRS.L3) + if (fs.existsSync(l3Path)) { + const content = fs.readFileSync(l3Path, "utf-8") + const lines = content.split("\n").length + const bytes = Buffer.byteLength(content, "utf-8") + + console.log() + console.log(`📂 L3 · persona`) + console.log(` 大小: ${formatBytes(bytes)} 行数: ${lines}`) + } else { + console.log() + console.log(`📂 L3 · persona （文件不存在）`) + } + + console.log() + console.log(`${"═".repeat(60)}`) + console.log(`💡 使用 -L <层级> 查看详细数据，如: -L L0 --since 7d`) + console.log() +} + +function formatBytes(bytes: number): string { + if (bytes < 1024) return `${bytes} B` + if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB` + return `${(bytes / (1024 * 1024)).toFixed(1)} MB` +} + +// ───────────────────────────────────────────── +// Main +// ───────────────────────────────────────────── + +/** 尝试打开 SQLite 数据库，不存在时返回 null */ +function tryOpenSqlite(dataDir: string): DatabaseSync | null { + const dbPath = path.join(dataDir, SQLITE_DB_NAME) + if (!fs.existsSync(dbPath)) { + return null + } + return openSqliteReadonly(dbPath) +} + +/** L0/L1 数据库不存在时返回空数据（正常业务场景：插件刚安装，尚未产生对话） */ +function emptyL0L1Result(opts: CliOptions, level: "L0" | "L1") { + if (opts.format === "json") { + const result: QueryResult> = { + level, + total: 0, + offset: opts.offset, + limit: opts.limit, + sort: opts.sort, + filter: null, + data: [], + } + console.log(JSON.stringify(result)) + return + } + if (opts.format === "jsonl") { + return + } + const label = level === "L0" ? "conversations" : "records" + console.log() + console.log(`📊 查询结果：${level} ${label}（SQLite）`) + console.log(` （数据库尚未生成，暂无数据）`) + console.log() +} + +function main() { + const opts = parseCli() + + // L2/L3 不依赖 SQLite 数据库，直接处理 + if (opts.level === "L2") { + queryL2(opts) + return + } + if (opts.level === "L3") { + queryL3(opts) + return + } + + // L0/L1/概览模式需要 SQLite + const db = tryOpenSqlite(opts.dataDir) + + // 数据库不存在：L0/L1 返回空数据，概览模式提示 + if (!db) { + if (opts.level === "L0" || opts.level === "L1") { + emptyL0L1Result(opts, opts.level) + return + } + // 概览模式：数据库不存在，报错退出 + console.error(`❌ SQLite 数据库不存在: ${path.join(opts.dataDir, SQLITE_DB_NAME)}`) + console.error(` 请确认数据目录下包含 ${SQLITE_DB_NAME}`) + process.exit(1) + } + + try { + if (!opts.level) { + showOverview(db, opts) + return + } + + switch (opts.level) { + case "L0": + querySqliteLevel(db, opts, "L0") + break + case "L1": + querySqliteLevel(db, opts, "L1") + break + } + } finally { + db.close() + } +} + +main() diff --git a/scripts/read-local-memory/tsconfig.json b/scripts/read-local-memory/tsconfig.json new file mode 100644 index 0000000..1a71655 --- /dev/null +++ b/scripts/read-local-memory/tsconfig.json @@ -0,0 +1,19 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "Node16", + "moduleResolution": "Node16", + "outDir": "./dist", + "rootDir": ".", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "resolveJsonModule": true, + "types": ["node"], + "declaration": false, + "sourceMap": false + }, + "include": ["read-local-memory.ts"], + "exclude": ["dist", "node_modules"] +} diff --git a/src/config.ts b/src/config.ts index 30a68e3..e09cff5 100644 --- a/src/config.ts +++ b/src/config.ts @@ -478,7 +478,7 @@ export function parseConfig(raw: Record | undefined): MemoryTda everyNConversations: num(pipelineGroup, "everyNConversations") ?? 5, enableWarmup: bool(pipelineGroup, "enableWarmup") ?? true, l1IdleTimeoutSeconds: num(pipelineGroup, "l1IdleTimeoutSeconds") ?? 600, - l2DelayAfterL1Seconds: num(pipelineGroup, "l2DelayAfterL1Seconds") ?? 90, + l2DelayAfterL1Seconds: num(pipelineGroup, "l2DelayAfterL1Seconds") ?? 10, l2MinIntervalSeconds: num(pipelineGroup, "l2MinIntervalSeconds") ?? 900, l2MaxIntervalSeconds: num(pipelineGroup, "l2MaxIntervalSeconds") ?? 3600, sessionActiveWindowHours: num(pipelineGroup, "sessionActiveWindowHours") ?? 24,