mirror of
https://github.com/TencentCloud/TencentDB-Agent-Memory
synced 2026-07-10 12:34:27 +00:00
chore: release v0.3.5
This commit is contained in:
chrishuan
@@ -0,0 +1,634 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* 腾讯云 VDB (Tencent VectorDB) 数据导出脚本
|
||||
*
|
||||
* 连接腾讯云向量数据库实例，查询指定数据库下 collection 的文档，导出为 .jsonl 文件。
|
||||
* 仅支持腾讯云向量数据库（Tencent VectorDB），不支持其他厂商的向量数据库。
|
||||
*
|
||||
* 所有连接参数通过 CLI 传入，无需 .env 文件。
|
||||
*
|
||||
* 用法：
|
||||
* node ./bin/export-tencent-vdb.mjs --url <地址> --username <用户名> --api-key <密钥> --database <库名>
|
||||
* node ./bin/export-tencent-vdb.mjs --url <地址> --username <用户名> --api-key <密钥> --database <库名> --probe
|
||||
* node ./bin/export-tencent-vdb.mjs --url <地址> --username <用户名> --api-key <密钥> --database <库名> -c <collection> -o /tmp/backup
|
||||
*
|
||||
* 输出：
|
||||
* 默认输出到当前工作目录下的 ./vdb-export-YYYY-MM-DD/，可通过 -o 指定。
|
||||
* <outputDir>/
|
||||
* ├── <collection>.jsonl — 每行一个 JSON 文档
|
||||
* ├── schemas.json — 导出的 collection 表结构（索引、embedding 配置等）
|
||||
* └── export-meta.json — 导出元信息
|
||||
*
|
||||
* 导出字段说明：
|
||||
* 默认行为：导出所有字段，但跳过 vector（稠密向量，1024维浮点数组，体积大）。
|
||||
* 加 --include-vectors：导出全部字段，包括 vector，不跳过任何内容。
|
||||
* 注：sparse_vector（BM25 稀疏向量）始终导出，不受此开关影响。
|
||||
*
|
||||
* 依赖：Node.js >= 18（内置 fetch）
|
||||
*/
|
||||
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
|
||||
// ============================================================
|
||||
// CLI 参数解析（含 VDB 连接信息）
|
||||
// ============================================================
|
||||
|
||||
interface VDBConfig {
|
||||
url: string;
|
||||
username: string;
|
||||
apiKey: string;
|
||||
database: string;
|
||||
timeout: number;
|
||||
}
|
||||
|
||||
interface CliArgs {
|
||||
// 连接参数
|
||||
url?: string;
|
||||
username?: string;
|
||||
apiKey?: string;
|
||||
database?: string;
|
||||
timeout: number;
|
||||
// 导出参数
|
||||
output: string;
|
||||
collection?: string;
|
||||
filter?: string;
|
||||
limit?: number;
|
||||
offset: number;
|
||||
includeVectors: boolean;
|
||||
probe: boolean;
|
||||
help: boolean;
|
||||
}
|
||||
|
||||
const PAGE_SIZE = 100;
|
||||
|
||||
function parseArgs(): CliArgs {
|
||||
const args = process.argv.slice(2);
|
||||
const result: CliArgs = {
|
||||
timeout: 30000,
|
||||
output: `./vdb-export-${new Date().toISOString().slice(0, 10)}`,
|
||||
offset: 0,
|
||||
includeVectors: false,
|
||||
probe: false,
|
||||
help: false,
|
||||
};
|
||||
|
||||
for (let i = 0; i < args.length; i++) {
|
||||
switch (args[i]) {
|
||||
case "--url":
|
||||
result.url = args[++i];
|
||||
break;
|
||||
case "--username":
|
||||
result.username = args[++i];
|
||||
break;
|
||||
case "--api-key":
|
||||
result.apiKey = args[++i];
|
||||
break;
|
||||
case "--database":
|
||||
result.database = args[++i];
|
||||
break;
|
||||
case "--timeout":
|
||||
result.timeout = parseInt(args[++i], 10) || 30000;
|
||||
break;
|
||||
case "--output":
|
||||
case "-o":
|
||||
result.output = args[++i];
|
||||
break;
|
||||
case "--collection":
|
||||
case "-c":
|
||||
result.collection = args[++i];
|
||||
break;
|
||||
case "--filter":
|
||||
case "-f":
|
||||
result.filter = args[++i];
|
||||
break;
|
||||
case "--limit":
|
||||
case "-l": {
|
||||
const v = parseInt(args[++i], 10);
|
||||
if (isNaN(v) || v < 1) {
|
||||
console.error(`❌ --limit 必须 >= 1，收到: ${args[i]}`);
|
||||
process.exit(1);
|
||||
}
|
||||
result.limit = v;
|
||||
break;
|
||||
}
|
||||
case "--offset": {
|
||||
const v = parseInt(args[++i], 10);
|
||||
if (isNaN(v) || v < 0) {
|
||||
console.error(`❌ --offset 必须 >= 0，收到: ${args[i]}`);
|
||||
process.exit(1);
|
||||
}
|
||||
result.offset = v;
|
||||
break;
|
||||
}
|
||||
case "--include-vectors":
|
||||
result.includeVectors = true;
|
||||
break;
|
||||
case "--probe":
|
||||
result.probe = true;
|
||||
break;
|
||||
case "--help":
|
||||
case "-h":
|
||||
result.help = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
function validateConfig(args: CliArgs): VDBConfig {
|
||||
const missing: string[] = [];
|
||||
if (!args.url) missing.push("--url");
|
||||
if (!args.username) missing.push("--username");
|
||||
if (!args.apiKey) missing.push("--api-key");
|
||||
if (!args.database) missing.push("--database");
|
||||
|
||||
if (missing.length > 0) {
|
||||
console.error("❌ 缺少必填参数：");
|
||||
for (const k of missing) {
|
||||
console.error(` - ${k}`);
|
||||
}
|
||||
console.error();
|
||||
console.error("示例：");
|
||||
console.error();
|
||||
console.error(' node ./bin/export-tencent-vdb.mjs \\');
|
||||
console.error(' --url "http://your-vdb-host:8100" \\');
|
||||
console.error(' --username "root" \\');
|
||||
console.error(' --api-key "your-api-key" \\');
|
||||
console.error(' --database "your-database"');
|
||||
console.error();
|
||||
console.error("使用 --help 查看完整参数说明。");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
return {
|
||||
url: args.url!,
|
||||
username: args.username!,
|
||||
apiKey: args.apiKey!,
|
||||
database: args.database!,
|
||||
timeout: args.timeout,
|
||||
};
|
||||
}
|
||||
|
||||
function printHelp(): void {
|
||||
console.log(`
|
||||
腾讯云 VDB (Tencent VectorDB) 数据导出脚本
|
||||
|
||||
用法：
|
||||
node ./bin/export-tencent-vdb.mjs [连接参数] [选项]
|
||||
|
||||
连接参数（必填）：
|
||||
--url <地址> VDB 实例 HTTP 地址（如 http://your-vdb-host:8100）
|
||||
--username <用户名> 认证用户名（如 root）
|
||||
--api-key <密钥> 认证密钥
|
||||
--database <库名> 数据库名称
|
||||
|
||||
选项：
|
||||
--timeout <毫秒> 单次请求超时（默认: 30000）
|
||||
-o, --output <目录> 输出目录（默认: ./vdb-export-YYYY-MM-DD）
|
||||
-c, --collection <全名> 只导出指定 collection（全名匹配，不指定则导出所有）
|
||||
-f, --filter <表达式> VDB Filter 过滤条件（如 'agent_id = "xxx"'）
|
||||
-l, --limit <数量> 最多导出多少条（不指定则导出全部）
|
||||
--offset <偏移> 从第几条开始（默认: 0），须为分页大小的整数倍
|
||||
--include-vectors 保留 vector 稠密向量字段（默认跳过）
|
||||
--probe 仅测试连通性，列出 collection 信息后退出
|
||||
-h, --help 显示帮助
|
||||
|
||||
输出：
|
||||
<outputDir>/
|
||||
├── <collection全名>.jsonl 每行一个 JSON 文档
|
||||
├── schemas.json 表结构
|
||||
└── export-meta.json 导出元信息
|
||||
|
||||
导出字段说明：
|
||||
默认跳过 vector（稠密向量），保留 sparse_vector（BM25）。
|
||||
加 --include-vectors 导出全部字段。
|
||||
|
||||
示例：
|
||||
# 测试连通性
|
||||
node ./bin/export-tencent-vdb.mjs \\
|
||||
--url "http://gz-vdb-xxx:8100" --username root --api-key "xxx" --database mydb \\
|
||||
--probe
|
||||
|
||||
# 全量导出
|
||||
node ./bin/export-tencent-vdb.mjs \\
|
||||
--url "http://gz-vdb-xxx:8100" --username root --api-key "xxx" --database mydb
|
||||
|
||||
# 导出指定 collection 到指定目录
|
||||
node ./bin/export-tencent-vdb.mjs \\
|
||||
--url "http://gz-vdb-xxx:8100" --username root --api-key "xxx" --database mydb \\
|
||||
-c mydb_l0_conversations -o /tmp/backup
|
||||
|
||||
# 带过滤条件
|
||||
node ./bin/export-tencent-vdb.mjs \\
|
||||
--url "http://gz-vdb-xxx:8100" --username root --api-key "xxx" --database mydb \\
|
||||
-f 'role = "user"'
|
||||
`);
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// VDB HTTP Client
|
||||
// ============================================================
|
||||
|
||||
class VDBClient {
|
||||
private baseUrl: string;
|
||||
private authHeader: string;
|
||||
private database: string;
|
||||
private timeout: number;
|
||||
|
||||
constructor(cfg: VDBConfig) {
|
||||
this.baseUrl = cfg.url.replace(/\/$/, "");
|
||||
this.authHeader = `Bearer account=${cfg.username}&api_key=${cfg.apiKey}`;
|
||||
this.database = cfg.database;
|
||||
this.timeout = cfg.timeout;
|
||||
}
|
||||
|
||||
async request<T>(apiPath: string, body: Record<string, unknown>): Promise<T> {
|
||||
const url = `${this.baseUrl}${apiPath}`;
|
||||
|
||||
const resp = await fetch(url, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
Authorization: this.authHeader,
|
||||
},
|
||||
body: JSON.stringify(body),
|
||||
signal: AbortSignal.timeout(this.timeout),
|
||||
});
|
||||
|
||||
if (!resp.ok) {
|
||||
const text = await resp.text().catch(() => "(unable to read body)");
|
||||
throw new Error(`VDB API error: HTTP ${resp.status} — ${text.slice(0, 500)}`);
|
||||
}
|
||||
|
||||
const json = (await resp.json()) as { code: number; msg: string } & T;
|
||||
if (json.code !== 0) {
|
||||
throw new Error(`VDB API error [${apiPath}]: code=${json.code}, msg=${json.msg}`);
|
||||
}
|
||||
return json;
|
||||
}
|
||||
|
||||
async listCollections(): Promise<
|
||||
Array<{ collection: string; documentCount: number }>
|
||||
> {
|
||||
const result = await this.request<{
|
||||
collections: Array<{
|
||||
collection: string;
|
||||
documentCount: number;
|
||||
[key: string]: unknown;
|
||||
}>;
|
||||
}>("/collection/list", {
|
||||
database: this.database,
|
||||
});
|
||||
return (result.collections || []).map((c) => ({
|
||||
collection: c.collection,
|
||||
documentCount: c.documentCount ?? 0,
|
||||
}));
|
||||
}
|
||||
|
||||
async queryDocuments(
|
||||
collection: string,
|
||||
options: {
|
||||
limit: number;
|
||||
offset: number;
|
||||
filter?: string;
|
||||
retrieveVector?: boolean;
|
||||
},
|
||||
): Promise<{
|
||||
documents: Array<Record<string, unknown>>;
|
||||
count: number;
|
||||
}> {
|
||||
const query: Record<string, unknown> = {
|
||||
limit: options.limit,
|
||||
offset: options.offset,
|
||||
};
|
||||
if (options.filter) {
|
||||
query.filter = options.filter;
|
||||
}
|
||||
if (options.retrieveVector) {
|
||||
query.retrieveVector = true;
|
||||
}
|
||||
|
||||
const result = await this.request<{
|
||||
documents: Array<Record<string, unknown>>;
|
||||
count: number;
|
||||
}>("/document/query", {
|
||||
database: this.database,
|
||||
collection,
|
||||
readConsistency: "strongConsistency",
|
||||
query,
|
||||
});
|
||||
|
||||
return {
|
||||
documents: result.documents || [],
|
||||
count: result.count ?? 0,
|
||||
};
|
||||
}
|
||||
|
||||
async describeCollection(collection: string): Promise<Record<string, unknown>> {
|
||||
const result = await this.request<{
|
||||
collection: Record<string, unknown>;
|
||||
}>("/collection/describe", {
|
||||
database: this.database,
|
||||
collection,
|
||||
});
|
||||
return result.collection || {};
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// 导出逻辑
|
||||
// ============================================================
|
||||
|
||||
interface ExportOptions {
|
||||
filter?: string;
|
||||
limit?: number;
|
||||
offset: number;
|
||||
includeVectors: boolean;
|
||||
expectedTotal?: number;
|
||||
}
|
||||
|
||||
async function exportCollection(
|
||||
client: VDBClient,
|
||||
collection: string,
|
||||
outputDir: string,
|
||||
options: ExportOptions,
|
||||
): Promise<{ docCount: number; filePath: string }> {
|
||||
const filePath = path.join(outputDir, `${collection}.jsonl`);
|
||||
const writeStream = fs.createWriteStream(filePath, { encoding: "utf-8" });
|
||||
|
||||
const isRangeMode = options.limit !== undefined;
|
||||
const maxDocs = options.limit ?? Infinity;
|
||||
const pageSize = isRangeMode ? Math.min(options.limit!, PAGE_SIZE) : PAGE_SIZE;
|
||||
|
||||
let currentOffset = options.offset;
|
||||
let totalExported = 0;
|
||||
let hasMore = true;
|
||||
|
||||
console.log(` 📦 ${collection}`);
|
||||
if (options.expectedTotal !== undefined) {
|
||||
console.log(` 文档总数: ${options.expectedTotal}`);
|
||||
}
|
||||
if (options.filter) {
|
||||
console.log(` 过滤条件: ${options.filter}`);
|
||||
}
|
||||
if (isRangeMode) {
|
||||
console.log(` 导出范围: offset=${options.offset}, limit=${options.limit}`);
|
||||
}
|
||||
|
||||
while (hasMore && totalExported < maxDocs) {
|
||||
const remaining = maxDocs - totalExported;
|
||||
const thisPageSize = Math.min(pageSize, remaining);
|
||||
|
||||
try {
|
||||
const result = await client.queryDocuments(collection, {
|
||||
limit: thisPageSize,
|
||||
offset: currentOffset,
|
||||
filter: options.filter,
|
||||
retrieveVector: options.includeVectors,
|
||||
});
|
||||
|
||||
const docs = result.documents;
|
||||
if (!docs || docs.length === 0) {
|
||||
hasMore = false;
|
||||
break;
|
||||
}
|
||||
|
||||
for (const doc of docs) {
|
||||
const exportDoc = { ...doc };
|
||||
if (!options.includeVectors) {
|
||||
delete exportDoc.vector;
|
||||
}
|
||||
writeStream.write(JSON.stringify(exportDoc) + "\n");
|
||||
}
|
||||
|
||||
totalExported += docs.length;
|
||||
currentOffset += docs.length;
|
||||
|
||||
if (options.expectedTotal !== undefined && !isRangeMode) {
|
||||
const pct = Math.min(
|
||||
100,
|
||||
Math.round((totalExported / options.expectedTotal) * 100),
|
||||
);
|
||||
process.stdout.write(
|
||||
`\r 进度: ${totalExported}/${options.expectedTotal} (${pct}%)`,
|
||||
);
|
||||
} else {
|
||||
process.stdout.write(`\r 已导出: ${totalExported} 条`);
|
||||
}
|
||||
|
||||
if (docs.length < thisPageSize) {
|
||||
hasMore = false;
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(
|
||||
`\n ❌ 查询失败 (offset=${currentOffset}): ${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
hasMore = false;
|
||||
}
|
||||
}
|
||||
|
||||
writeStream.end();
|
||||
await new Promise<void>((resolve) => writeStream.on("finish", resolve));
|
||||
|
||||
console.log(
|
||||
`\n ✅ 完成: ${totalExported} 条 → ${path.basename(filePath)}`,
|
||||
);
|
||||
|
||||
return { docCount: totalExported, filePath };
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// Main
|
||||
// ============================================================
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const args = parseArgs();
|
||||
|
||||
if (args.help) {
|
||||
printHelp();
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
const config = validateConfig(args);
|
||||
|
||||
console.log("╔═══════════════════════════════════════════════════╗");
|
||||
console.log("║ 腾讯云 VDB (Tencent VectorDB) 数据导出工具 ║");
|
||||
console.log("╚═══════════════════════════════════════════════════╝");
|
||||
console.log();
|
||||
console.log(`📌 VDB 地址: ${config.url}`);
|
||||
console.log(`📌 数据库: ${config.database}`);
|
||||
console.log(`📌 输出目录: ${args.output}`);
|
||||
if (args.collection) {
|
||||
console.log(`📌 指定导出: ${args.collection}`);
|
||||
}
|
||||
if (args.filter) {
|
||||
console.log(`📌 过滤条件: ${args.filter}`);
|
||||
}
|
||||
if (args.limit !== undefined) {
|
||||
console.log(`📌 导出上限: ${args.limit} 条`);
|
||||
}
|
||||
if (args.offset > 0) {
|
||||
console.log(`📌 起始偏移: ${args.offset}`);
|
||||
}
|
||||
if (args.includeVectors) {
|
||||
console.log(`📌 包含向量: 是`);
|
||||
}
|
||||
console.log();
|
||||
|
||||
fs.mkdirSync(args.output, { recursive: true });
|
||||
|
||||
const client = new VDBClient(config);
|
||||
|
||||
let allCollections: Array<{ collection: string; documentCount: number }>;
|
||||
try {
|
||||
allCollections = await client.listCollections();
|
||||
} catch (err) {
|
||||
console.error(
|
||||
`❌ 列出 collection 失败: ${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
let targetCollections: Array<{ collection: string; documentCount: number }>;
|
||||
if (args.collection) {
|
||||
const found = allCollections.find((c) => c.collection === args.collection);
|
||||
if (!found) {
|
||||
console.error(
|
||||
`❌ Collection "${args.collection}" 不存在。可用的 collection：`,
|
||||
);
|
||||
for (const c of allCollections) {
|
||||
console.error(` - ${c.collection} (${c.documentCount} 条)`);
|
||||
}
|
||||
process.exit(1);
|
||||
}
|
||||
targetCollections = [found];
|
||||
} else {
|
||||
targetCollections = allCollections;
|
||||
console.log(
|
||||
`🔍 找到 ${targetCollections.length} 个 collection：`,
|
||||
);
|
||||
for (const c of targetCollections) {
|
||||
console.log(` - ${c.collection} (${c.documentCount} 条)`);
|
||||
}
|
||||
}
|
||||
|
||||
if (targetCollections.length === 0) {
|
||||
console.log("⚠️ 数据库中没有 collection，无数据可导出");
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
// --probe 模式：只测试连通性，列出信息后退出
|
||||
if (args.probe) {
|
||||
console.log();
|
||||
console.log("✅ 连通性测试通过");
|
||||
console.log();
|
||||
console.log(` VDB 地址: ${config.url}`);
|
||||
console.log(` 数据库: ${config.database}`);
|
||||
console.log(` Collection: ${targetCollections.length} 个`);
|
||||
const totalDocs = targetCollections.reduce((s, c) => s + c.documentCount, 0);
|
||||
console.log(` 总文档数: ${totalDocs}`);
|
||||
console.log();
|
||||
for (const c of targetCollections) {
|
||||
console.log(` - ${c.collection} (${c.documentCount} 条)`);
|
||||
}
|
||||
console.log();
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
console.log();
|
||||
|
||||
// 获取并保存表结构
|
||||
const schemas: Record<string, Record<string, unknown>> = {};
|
||||
console.log("📐 获取表结构...");
|
||||
for (const col of targetCollections) {
|
||||
try {
|
||||
const schema = await client.describeCollection(col.collection);
|
||||
schemas[col.collection] = schema;
|
||||
const indexCount = Array.isArray(schema.indexes) ? schema.indexes.length : 0;
|
||||
const emb = schema.embedding as Record<string, unknown> | undefined;
|
||||
const embInfo = emb ? `embedding=${emb.field}→${emb.model}` : "无 embedding";
|
||||
console.log(` ✅ ${col.collection} (${indexCount} 个索引, ${embInfo})`);
|
||||
} catch (err) {
|
||||
console.error(
|
||||
` ⚠️ ${col.collection} 表结构获取失败: ${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
console.log();
|
||||
|
||||
const schemaPath = path.join(args.output, "schemas.json");
|
||||
fs.writeFileSync(schemaPath, JSON.stringify(schemas, null, 2) + "\n");
|
||||
|
||||
const exportResults: Array<{
|
||||
collection: string;
|
||||
docCount: number;
|
||||
filePath: string;
|
||||
}> = [];
|
||||
|
||||
for (const col of targetCollections) {
|
||||
try {
|
||||
const result = await exportCollection(client, col.collection, args.output, {
|
||||
filter: args.filter,
|
||||
limit: args.limit,
|
||||
offset: args.offset,
|
||||
includeVectors: args.includeVectors,
|
||||
expectedTotal: col.documentCount,
|
||||
});
|
||||
exportResults.push({ collection: col.collection, ...result });
|
||||
} catch (err) {
|
||||
console.error(
|
||||
`❌ 导出 ${col.collection} 失败: ${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
exportResults.push({
|
||||
collection: col.collection,
|
||||
docCount: 0,
|
||||
filePath: "",
|
||||
});
|
||||
}
|
||||
console.log();
|
||||
}
|
||||
|
||||
const meta = {
|
||||
exportedAt: new Date().toISOString(),
|
||||
vdbUrl: config.url,
|
||||
database: config.database,
|
||||
filter: args.filter ?? null,
|
||||
offset: args.offset,
|
||||
limit: args.limit ?? null,
|
||||
includeVectors: args.includeVectors,
|
||||
collections: exportResults.map((r) => ({
|
||||
collection: r.collection,
|
||||
documentCount: r.docCount,
|
||||
file: r.filePath ? path.basename(r.filePath) : null,
|
||||
})),
|
||||
totalDocuments: exportResults.reduce((sum, r) => sum + r.docCount, 0),
|
||||
};
|
||||
|
||||
const metaPath = path.join(args.output, "export-meta.json");
|
||||
fs.writeFileSync(metaPath, JSON.stringify(meta, null, 2) + "\n");
|
||||
|
||||
console.log("═══════════════════════════════════════════════════");
|
||||
console.log(" ✅ 导出完成");
|
||||
console.log("═══════════════════════════════════════════════════");
|
||||
console.log();
|
||||
console.log(` 📁 输出目录: ${args.output}`);
|
||||
console.log(` 📊 总文档数: ${meta.totalDocuments}`);
|
||||
for (const r of exportResults) {
|
||||
const status = r.docCount > 0 ? "✅" : "⚠️";
|
||||
console.log(
|
||||
` ${status} ${r.collection}: ${r.docCount} 条`,
|
||||
);
|
||||
}
|
||||
console.log(` 📋 元信息: ${path.basename(metaPath)}`);
|
||||
console.log(` 📐 表结构: ${path.basename(schemaPath)}`);
|
||||
console.log();
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error(
|
||||
`\n❌ 导出失败: ${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2022",
|
||||
"module": "Node16",
|
||||
"moduleResolution": "Node16",
|
||||
"outDir": "./dist",
|
||||
"rootDir": ".",
|
||||
"strict": true,
|
||||
"esModuleInterop": true,
|
||||
"skipLibCheck": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"resolveJsonModule": true,
|
||||
"types": ["node"],
|
||||
"declaration": false,
|
||||
"sourceMap": false
|
||||
},
|
||||
"include": ["export-tencent-vdb.ts"],
|
||||
"exclude": ["dist", "node_modules", "docs"]
|
||||
}
|
||||
@@ -0,0 +1,239 @@
|
||||
# SQLite → 腾讯云向量数据库迁移工具
|
||||
|
||||
离线迁移工具，用于将 memory-tdai 的数据从本地 SQLite 存储迁移到腾讯云向量数据库（TCVDB）。
|
||||
|
||||
## 前置条件
|
||||
|
||||
- Node.js >= 22.16.0
|
||||
- 插件已通过 `openclaw plugins install` 安装
|
||||
- 迁移脚本已编译（见下方）
|
||||
|
||||
## 编译
|
||||
|
||||
迁移脚本使用 TypeScript 编写，运行前需要先编译：
|
||||
|
||||
```bash
|
||||
npm run build:migrate-sqlite-to-vdb
|
||||
```
|
||||
|
||||
编译产物输出到 `scripts/migrate-sqlite-to-tcvdb/dist/`，可直接用 Node 运行。
|
||||
|
||||
## 使用方法
|
||||
|
||||
```bash
|
||||
# 预检模式（仅查看源数据，不执行写入）
|
||||
npm run migrate:sqlite-to-tcvdb -- \
|
||||
--plugin-data-dir ~/.openclaw/memory-tdai \
|
||||
--openclaw-config-path ~/.openclaw/openclaw.json \
|
||||
--tcvdb-url http://127.0.0.1:80 \
|
||||
--tcvdb-username root \
|
||||
--tcvdb-api-key-env TCVDB_API_KEY \
|
||||
--tcvdb-database agent_memory_prod \
|
||||
--tcvdb-embedding-model bge-large-zh \
|
||||
--dry-run
|
||||
|
||||
# 正式迁移
|
||||
npm run migrate:sqlite-to-tcvdb -- \
|
||||
--plugin-data-dir ~/.openclaw/memory-tdai \
|
||||
--openclaw-config-path ~/.openclaw/openclaw.json \
|
||||
--tcvdb-url http://127.0.0.1:80 \
|
||||
--tcvdb-username root \
|
||||
--tcvdb-api-key-env TCVDB_API_KEY \
|
||||
--tcvdb-database agent_memory_prod \
|
||||
--tcvdb-embedding-model bge-large-zh \
|
||||
--yes
|
||||
```
|
||||
|
||||
### 更多示例
|
||||
|
||||
```bash
|
||||
# 直接传入 API Key（不通过环境变量）
|
||||
npm run migrate:sqlite-to-tcvdb -- \
|
||||
--plugin-data-dir ~/.openclaw/memory-tdai \
|
||||
--openclaw-config-path ~/.openclaw/openclaw.json \
|
||||
--tcvdb-url http://127.0.0.1:80 \
|
||||
--tcvdb-username root \
|
||||
--tcvdb-api-key 'your-api-key-here' \
|
||||
--tcvdb-database agent_memory_prod \
|
||||
--tcvdb-embedding-model bge-large-zh \
|
||||
--yes
|
||||
```
|
||||
|
||||
```bash
|
||||
# 指定自定义 SQLite 路径（数据库不在默认 vectors.db 位置时）
|
||||
npm run migrate:sqlite-to-tcvdb -- \
|
||||
--plugin-data-dir ~/.openclaw/memory-tdai \
|
||||
--sqlite-path /backup/2026-04/vectors-snapshot.db \
|
||||
--openclaw-config-path ~/.openclaw/openclaw.json \
|
||||
--tcvdb-url http://127.0.0.1:80 \
|
||||
--tcvdb-username root \
|
||||
--tcvdb-api-key-env TCVDB_API_KEY \
|
||||
--tcvdb-database agent_memory_prod \
|
||||
--tcvdb-embedding-model bge-large-zh \
|
||||
--yes
|
||||
```
|
||||
|
||||
```bash
|
||||
# 只迁移 L1 记忆层（跳过 L0 原始消息和 Profile）
|
||||
npm run migrate:sqlite-to-tcvdb -- \
|
||||
--plugin-data-dir ~/.openclaw/memory-tdai \
|
||||
--openclaw-config-path ~/.openclaw/openclaw.json \
|
||||
--tcvdb-url http://127.0.0.1:80 \
|
||||
--tcvdb-username root \
|
||||
--tcvdb-api-key-env TCVDB_API_KEY \
|
||||
--tcvdb-database agent_memory_prod \
|
||||
--tcvdb-embedding-model bge-large-zh \
|
||||
--layers l1 \
|
||||
--yes
|
||||
```
|
||||
|
||||
```bash
|
||||
# 只迁移 L0 和 L1（不迁移 Profile）
|
||||
npm run migrate:sqlite-to-tcvdb -- \
|
||||
--plugin-data-dir ~/.openclaw/memory-tdai \
|
||||
--openclaw-config-path ~/.openclaw/openclaw.json \
|
||||
--tcvdb-url http://127.0.0.1:80 \
|
||||
--tcvdb-username root \
|
||||
--tcvdb-api-key-env TCVDB_API_KEY \
|
||||
--tcvdb-database agent_memory_prod \
|
||||
--tcvdb-embedding-model bge-large-zh \
|
||||
--layers l0,l1 \
|
||||
--yes
|
||||
```
|
||||
|
||||
```bash
|
||||
# 英文语料场景：使用英文 BM25 分词
|
||||
npm run migrate:sqlite-to-tcvdb -- \
|
||||
--plugin-data-dir ~/.openclaw/memory-tdai \
|
||||
--openclaw-config-path ~/.openclaw/openclaw.json \
|
||||
--tcvdb-url http://127.0.0.1:80 \
|
||||
--tcvdb-username root \
|
||||
--tcvdb-api-key-env TCVDB_API_KEY \
|
||||
--tcvdb-database agent_memory_prod \
|
||||
--tcvdb-embedding-model bge-large-en-v1.5 \
|
||||
--bm25-language en \
|
||||
--yes
|
||||
```
|
||||
|
||||
```bash
|
||||
# 禁用 BM25 稀疏向量（仅使用密集向量检索）
|
||||
npm run migrate:sqlite-to-tcvdb -- \
|
||||
--plugin-data-dir ~/.openclaw/memory-tdai \
|
||||
--openclaw-config-path ~/.openclaw/openclaw.json \
|
||||
--tcvdb-url http://127.0.0.1:80 \
|
||||
--tcvdb-username root \
|
||||
--tcvdb-api-key-env TCVDB_API_KEY \
|
||||
--tcvdb-database agent_memory_prod \
|
||||
--tcvdb-embedding-model bge-large-zh \
|
||||
--no-bm25-enabled \
|
||||
--yes
|
||||
```
|
||||
|
||||
```bash
|
||||
# 仅迁移数据，不自动更新 openclaw.json 和 manifest（手动管理配置）
|
||||
npm run migrate:sqlite-to-tcvdb -- \
|
||||
--plugin-data-dir ~/.openclaw/memory-tdai \
|
||||
--openclaw-config-path ~/.openclaw/openclaw.json \
|
||||
--tcvdb-url http://127.0.0.1:80 \
|
||||
--tcvdb-username root \
|
||||
--tcvdb-api-key-env TCVDB_API_KEY \
|
||||
--tcvdb-database agent_memory_prod \
|
||||
--tcvdb-embedding-model bge-large-zh \
|
||||
--no-apply-config \
|
||||
--no-rewrite-manifest \
|
||||
--yes
|
||||
```
|
||||
|
||||
```bash
|
||||
# 追加迁移：允许目标库已有数据，跳过非空检查
|
||||
npm run migrate:sqlite-to-tcvdb -- \
|
||||
--plugin-data-dir ~/.openclaw/memory-tdai \
|
||||
--openclaw-config-path ~/.openclaw/openclaw.json \
|
||||
--tcvdb-url http://127.0.0.1:80 \
|
||||
--tcvdb-username root \
|
||||
--tcvdb-api-key-env TCVDB_API_KEY \
|
||||
--tcvdb-database agent_memory_prod \
|
||||
--tcvdb-embedding-model bge-large-zh \
|
||||
--no-fail-if-target-nonempty \
|
||||
--no-verify-counts \
|
||||
--yes
|
||||
```
|
||||
|
||||
```bash
|
||||
# 输出迁移摘要到 JSON 文件（适合 CI/自动化流水线）
|
||||
npm run migrate:sqlite-to-tcvdb -- \
|
||||
--plugin-data-dir ~/.openclaw/memory-tdai \
|
||||
--openclaw-config-path ~/.openclaw/openclaw.json \
|
||||
--tcvdb-url http://127.0.0.1:80 \
|
||||
--tcvdb-username root \
|
||||
--tcvdb-api-key-env TCVDB_API_KEY \
|
||||
--tcvdb-database agent_memory_prod \
|
||||
--tcvdb-embedding-model bge-large-zh \
|
||||
--summary-json-path ./migration-report.json \
|
||||
--job-id "migrate-2026-04-13" \
|
||||
--yes
|
||||
```
|
||||
|
||||
```bash
|
||||
# 设置自定义超时和别名
|
||||
npm run migrate:sqlite-to-tcvdb -- \
|
||||
--plugin-data-dir ~/.openclaw/memory-tdai \
|
||||
--openclaw-config-path ~/.openclaw/openclaw.json \
|
||||
--tcvdb-url http://10.0.1.50:80 \
|
||||
--tcvdb-username admin \
|
||||
--tcvdb-api-key-env TCVDB_API_KEY \
|
||||
--tcvdb-database agent_memory_prod \
|
||||
--tcvdb-embedding-model bge-large-zh \
|
||||
--tcvdb-alias "生产环境-主库" \
|
||||
--tcvdb-timeout-ms 30000 \
|
||||
--yes
|
||||
```
|
||||
|
||||
## 参数说明
|
||||
|
||||
| 参数 | 必填 | 默认值 | 说明 |
|
||||
|---|---|---|---|
|
||||
| `--plugin-data-dir` | 是 | — | 插件数据目录路径 |
|
||||
| `--openclaw-config-path` | 是 | — | `openclaw.json` 配置文件路径 |
|
||||
| `--sqlite-path` | 否 | `<plugin-data-dir>/vectors.db` | SQLite 数据库文件路径（默认取数据目录下的 `vectors.db`） |
|
||||
| `--plugin-id` | 否 | `memory-tencentdb` | 写入配置时使用的插件 ID |
|
||||
| `--tcvdb-url` | 是 | — | TCVDB 服务地址 |
|
||||
| `--tcvdb-username` | 是 | — | TCVDB 用户名 |
|
||||
| `--tcvdb-api-key` | * | — | TCVDB API 密钥（明文） |
|
||||
| `--tcvdb-api-key-env` | * | — | 包含 API 密钥的环境变量名 |
|
||||
| `--tcvdb-database` | 是 | — | TCVDB 数据库名 |
|
||||
| `--tcvdb-embedding-model` | 是 | — | Embedding 模型名称 |
|
||||
| `--tcvdb-alias` | 否 | `""` | 用户自定义别名 |
|
||||
| `--tcvdb-timeout-ms` | 否 | `10000` | 请求超时时间（毫秒） |
|
||||
| `--layers` | 否 | `l0,l1,l2,l3` | 要迁移的层（逗号分隔） |
|
||||
| `--dry-run` | 否 | `false` | 仅预览，不执行写入 |
|
||||
| `--yes` | 否 | `false` | 跳过交互确认 |
|
||||
| `--apply-config` | 否 | `true` | 迁移后更新 openclaw.json |
|
||||
| `--config-backup` | 否 | `true` | 写入配置前先备份原配置文件 |
|
||||
| `--rewrite-manifest` | 否 | `true` | 将 manifest.json 更新为 tcvdb |
|
||||
| `--fail-if-target-nonempty` | 否 | `true` | 目标库非空时中止 |
|
||||
| `--verify-counts` | 否 | `true` | 迁移后校验记录数 |
|
||||
| `--summary-json-path` | 否 | — | 将迁移摘要写入此文件 |
|
||||
| `--job-id` | 否 | — | 迁移任务 ID（用于追踪） |
|
||||
| `--bm25-enabled` | 否 | `true` | 启用 BM25 稀疏向量 |
|
||||
| `--bm25-language` | 否 | `zh` | BM25 语言（`zh` 或 `en`） |
|
||||
|
||||
\* `--tcvdb-api-key` 和 `--tcvdb-api-key-env` 二选一，必须提供其中一个。
|
||||
|
||||
## 目录结构
|
||||
|
||||
```
|
||||
scripts/migrate-sqlite-to-tcvdb/
|
||||
├── cli-entry.ts # CLI 入口
|
||||
├── sqlite-to-tcvdb.ts # 迁移核心逻辑（参数解析、预检、数据迁移）
|
||||
├── config-write.ts # OpenClaw 配置更新（JSON5，自包含）
|
||||
├── manifest-write.ts # Manifest 重写
|
||||
├── *.test.ts # 就近放置的测试文件
|
||||
├── tsconfig.json # 迁移脚本编译配置
|
||||
├── dist/ # 编译产物（已 gitignore）
|
||||
└── README.md # 本文件
|
||||
|
||||
bin/migrate-sqlite-to-tcvdb.mjs # 极薄 bin 包装 → dist/
|
||||
```
|
||||
|
||||
迁移脚本通过 `../../src/` 引用存储实现（VectorStore、TcvdbMemoryStore 等），但**不依赖 `openclaw/plugin-sdk`**。配置写回直接使用 `json5`。
|
||||
@@ -0,0 +1,12 @@
|
||||
import { runMigrationCli } from "./sqlite-to-tcvdb.js";
|
||||
|
||||
const TAG = "[memory-tdai][migrate-cli]";
|
||||
|
||||
try {
|
||||
const summary = await runMigrationCli(process.argv.slice(2));
|
||||
process.stdout.write(`${JSON.stringify(summary, null, 2)}\n`);
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
process.stderr.write(`${TAG} ${message}\n`);
|
||||
process.exitCode = 1;
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
import fs from "node:fs/promises";
|
||||
import JSON5 from "json5";
|
||||
|
||||
export type Bm25Language = "zh" | "en";
|
||||
|
||||
export interface MigrationPluginConfigTarget {
|
||||
url: string;
|
||||
username: string;
|
||||
apiKey: string;
|
||||
database: string;
|
||||
alias: string;
|
||||
embeddingModel: string;
|
||||
timeout: number;
|
||||
bm25Enabled: boolean;
|
||||
bm25Language: Bm25Language;
|
||||
}
|
||||
|
||||
export interface WriteMigrationPluginConfigParams {
|
||||
configPath: string;
|
||||
pluginId: string;
|
||||
tcvdb: {
|
||||
url: string;
|
||||
username: string;
|
||||
apiKey: string;
|
||||
database: string;
|
||||
alias: string;
|
||||
embeddingModel: string;
|
||||
timeout: number;
|
||||
};
|
||||
bm25: {
|
||||
enabled: boolean;
|
||||
language: Bm25Language;
|
||||
};
|
||||
}
|
||||
|
||||
interface ConfigWriteAdapterDeps {
|
||||
fs?: Pick<typeof fs, "readFile" | "writeFile" | "mkdir">;
|
||||
parseConfig?: (raw: string) => unknown;
|
||||
}
|
||||
|
||||
function asRecord(value: unknown): Record<string, unknown> {
|
||||
return value !== null && typeof value === "object" && !Array.isArray(value)
|
||||
? { ...(value as Record<string, unknown>) }
|
||||
: {};
|
||||
}
|
||||
|
||||
export function buildMigrationPluginConfigPatch(target: MigrationPluginConfigTarget): Record<string, unknown> {
|
||||
return {
|
||||
storeBackend: "tcvdb",
|
||||
tcvdb: {
|
||||
url: target.url,
|
||||
username: target.username,
|
||||
apiKey: target.apiKey,
|
||||
database: target.database,
|
||||
alias: target.alias,
|
||||
embeddingModel: target.embeddingModel,
|
||||
timeout: target.timeout,
|
||||
},
|
||||
bm25: {
|
||||
enabled: target.bm25Enabled,
|
||||
language: target.bm25Language,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function applyPluginConfigPatch(
|
||||
sourceConfig: Record<string, unknown>,
|
||||
pluginId: string,
|
||||
patch: Record<string, unknown>,
|
||||
): Record<string, unknown> {
|
||||
const nextConfig = asRecord(sourceConfig);
|
||||
const plugins = asRecord(nextConfig.plugins);
|
||||
const entries = asRecord(plugins.entries);
|
||||
const targetEntry = asRecord(entries[pluginId]);
|
||||
const targetPluginConfig = asRecord(targetEntry.config);
|
||||
const patchTcvdb = asRecord(patch.tcvdb);
|
||||
const patchBm25 = asRecord(patch.bm25);
|
||||
|
||||
const mergedPluginConfig: Record<string, unknown> = {
|
||||
...targetPluginConfig,
|
||||
...patch,
|
||||
tcvdb: {
|
||||
...asRecord(targetPluginConfig.tcvdb),
|
||||
...patchTcvdb,
|
||||
},
|
||||
bm25: {
|
||||
...asRecord(targetPluginConfig.bm25),
|
||||
...patchBm25,
|
||||
},
|
||||
};
|
||||
|
||||
entries[pluginId] = {
|
||||
...targetEntry,
|
||||
config: mergedPluginConfig,
|
||||
};
|
||||
|
||||
plugins.entries = entries;
|
||||
nextConfig.plugins = plugins;
|
||||
return nextConfig;
|
||||
}
|
||||
|
||||
export async function writeMigrationPluginConfig(
|
||||
params: WriteMigrationPluginConfigParams,
|
||||
deps: ConfigWriteAdapterDeps = {},
|
||||
): Promise<void> {
|
||||
const fsImpl = deps.fs ?? fs;
|
||||
const parseConfig = deps.parseConfig ?? ((raw: string) => JSON5.parse(raw));
|
||||
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = parseConfig(await fsImpl.readFile(params.configPath, "utf-8"));
|
||||
} catch {
|
||||
throw new Error(
|
||||
`Config migration writer only supports single-file JSON/JSON5: ${params.configPath}`,
|
||||
);
|
||||
}
|
||||
|
||||
const nextConfig = applyPluginConfigPatch(
|
||||
asRecord(parsed),
|
||||
params.pluginId,
|
||||
buildMigrationPluginConfigPatch({
|
||||
...params.tcvdb,
|
||||
bm25Enabled: params.bm25.enabled,
|
||||
bm25Language: params.bm25.language,
|
||||
}),
|
||||
);
|
||||
|
||||
await fsImpl.writeFile(params.configPath, `${JSON.stringify(nextConfig, null, 2)}\n`, "utf-8");
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import {
|
||||
buildStoreInfo,
|
||||
manifestPath,
|
||||
readManifest,
|
||||
writeManifest,
|
||||
} from "../../src/utils/manifest.js";
|
||||
|
||||
export interface RewriteMigrationManifestParams {
|
||||
dataDir: string;
|
||||
tcvdbUrl: string;
|
||||
tcvdbDatabase: string;
|
||||
tcvdbAlias?: string;
|
||||
backupExisting?: boolean;
|
||||
now?: () => string;
|
||||
}
|
||||
|
||||
export interface RewriteMigrationManifestResult {
|
||||
created: boolean;
|
||||
updated: boolean;
|
||||
backupPath?: string;
|
||||
}
|
||||
|
||||
const DEFAULT_BACKUP_FILE = "manifest.json.migrate.bak";
|
||||
|
||||
export async function rewriteMigrationManifest(
|
||||
params: RewriteMigrationManifestParams,
|
||||
): Promise<RewriteMigrationManifestResult> {
|
||||
const existing = readManifest(params.dataDir);
|
||||
const nextStore = buildStoreInfo({
|
||||
type: "tcvdb",
|
||||
tcvdbUrl: params.tcvdbUrl,
|
||||
tcvdbDatabase: params.tcvdbDatabase,
|
||||
tcvdbAlias: params.tcvdbAlias,
|
||||
});
|
||||
|
||||
if (!existing) {
|
||||
writeManifest(params.dataDir, {
|
||||
version: 1,
|
||||
createdAt: params.now?.() ?? new Date().toISOString(),
|
||||
store: nextStore,
|
||||
seed: null,
|
||||
});
|
||||
return {
|
||||
created: true,
|
||||
updated: false,
|
||||
backupPath: undefined,
|
||||
};
|
||||
}
|
||||
|
||||
let backupPath: string | undefined;
|
||||
if (params.backupExisting !== false) {
|
||||
backupPath = path.join(path.dirname(manifestPath(params.dataDir)), DEFAULT_BACKUP_FILE);
|
||||
await fs.mkdir(path.dirname(backupPath), { recursive: true });
|
||||
await fs.copyFile(manifestPath(params.dataDir), backupPath);
|
||||
}
|
||||
|
||||
writeManifest(params.dataDir, {
|
||||
...existing,
|
||||
store: nextStore,
|
||||
});
|
||||
|
||||
return {
|
||||
created: false,
|
||||
updated: true,
|
||||
backupPath,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
/**
|
||||
* Type stub for node-llama-cpp — the migration script does not use local
|
||||
* embedding but TypeScript still resolves the module through transitive
|
||||
* imports (sqlite.ts → embedding.ts → import("node-llama-cpp")).
|
||||
*
|
||||
* This stub satisfies the compiler without requiring the actual package.
|
||||
*/
|
||||
declare module "node-llama-cpp" {
|
||||
const _: any;
|
||||
export = _;
|
||||
}
|
||||
@@ -0,0 +1,849 @@
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { parseArgs } from "node:util";
|
||||
import type { MemoryRecord } from "../../src/core/record/l1-writer.js";
|
||||
import { listLocalProfiles } from "../../src/core/profile/profile-sync.js";
|
||||
import { createBM25Encoder } from "../../src/core/store/bm25-local.js";
|
||||
import { VectorStore, type L0RecordRow, type L1RecordRow } from "../../src/core/store/sqlite.js";
|
||||
import { TcvdbMemoryStore } from "../../src/core/store/tcvdb.js";
|
||||
import type { L0Record, ProfileRecord, ProfileSyncRecord, StoreInitResult } from "../../src/core/store/types.js";
|
||||
import { readManifest } from "../../src/utils/manifest.js";
|
||||
import {
|
||||
rewriteMigrationManifest as rewriteMigrationManifestDefault,
|
||||
type RewriteMigrationManifestResult,
|
||||
} from "./manifest-write.js";
|
||||
import {
|
||||
writeMigrationPluginConfig as writeMigrationPluginConfigDefault,
|
||||
} from "./config-write.js";
|
||||
|
||||
export const DEFAULT_MIGRATION_PLUGIN_ID = "memory-tencentdb";
|
||||
export const ALL_MIGRATION_LAYERS = ["l0", "l1", "l2", "l3"] as const;
|
||||
|
||||
const TAG = "[memory-tdai][migrate]";
|
||||
|
||||
function log(message: string): void {
|
||||
process.stderr.write(`${TAG} ${message}\n`);
|
||||
}
|
||||
|
||||
function sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
export type MigrationLayer = (typeof ALL_MIGRATION_LAYERS)[number];
|
||||
export type Bm25Language = "zh" | "en";
|
||||
|
||||
export interface ResolvedMigrationCliOptions {
|
||||
pluginDataDir: string;
|
||||
sqlitePath: string;
|
||||
openclawConfigPath: string;
|
||||
pluginId: string;
|
||||
layers: MigrationLayer[];
|
||||
applyConfig: boolean;
|
||||
configBackup: boolean;
|
||||
rewriteManifest: boolean;
|
||||
failIfTargetNonempty: boolean;
|
||||
verifyCounts: boolean;
|
||||
dryRun: boolean;
|
||||
yes: boolean;
|
||||
bm25Enabled: boolean;
|
||||
bm25Language: Bm25Language;
|
||||
summaryJsonPath?: string;
|
||||
jobId?: string;
|
||||
tcvdb: {
|
||||
url: string;
|
||||
username: string;
|
||||
apiKey: string;
|
||||
database: string;
|
||||
alias: string;
|
||||
embeddingModel: string;
|
||||
timeout: number;
|
||||
caPemPath?: string;
|
||||
};
|
||||
}
|
||||
|
||||
export interface MigrationPreflightSummary {
|
||||
pluginId: string;
|
||||
dryRun: boolean;
|
||||
layers: MigrationLayer[];
|
||||
paths: {
|
||||
pluginDataDir: string;
|
||||
sqlitePath: string;
|
||||
openclawConfigPath: string;
|
||||
};
|
||||
source: {
|
||||
l0Count: number;
|
||||
l1Count: number;
|
||||
profileCount: number;
|
||||
manifestExists: boolean;
|
||||
manifestStoreType: "sqlite" | "tcvdb" | null;
|
||||
};
|
||||
target: {
|
||||
url: string;
|
||||
username: string;
|
||||
database: string;
|
||||
alias: string;
|
||||
embeddingModel: string;
|
||||
timeout: number;
|
||||
bm25Enabled: boolean;
|
||||
bm25Language: Bm25Language;
|
||||
};
|
||||
options: {
|
||||
applyConfig: boolean;
|
||||
configBackup: boolean;
|
||||
rewriteManifest: boolean;
|
||||
failIfTargetNonempty: boolean;
|
||||
verifyCounts: boolean;
|
||||
yes: boolean;
|
||||
};
|
||||
migration?: {
|
||||
l0Migrated: number;
|
||||
l1Migrated: number;
|
||||
profileMigrated: number;
|
||||
targetL0Count: number;
|
||||
targetL1Count: number;
|
||||
targetProfileCount: number;
|
||||
configWritten: boolean;
|
||||
manifestWritten: boolean;
|
||||
manifestBackupPath?: string;
|
||||
};
|
||||
}
|
||||
|
||||
export const DEFAULT_MIGRATION_PAGE_SIZE = 50;
|
||||
|
||||
export interface MigrationTargetStore {
|
||||
init(providerInfo?: unknown): Promise<StoreInitResult> | StoreInitResult;
|
||||
isDegraded(): boolean;
|
||||
close(): void;
|
||||
upsertL1(record: MemoryRecord, embedding?: Float32Array): Promise<boolean> | boolean;
|
||||
upsertL0(record: L0Record, embedding?: Float32Array): Promise<boolean> | boolean;
|
||||
upsertL1Batch?(records: MemoryRecord[]): Promise<number>;
|
||||
upsertL0Batch?(records: L0Record[]): Promise<number>;
|
||||
countL1(): Promise<number> | number;
|
||||
countL0(): Promise<number> | number;
|
||||
pullProfiles?(): Promise<ProfileRecord[]>;
|
||||
syncProfiles?(records: ProfileSyncRecord[]): Promise<void>;
|
||||
}
|
||||
|
||||
export interface RunMigrationCliDeps {
|
||||
createTargetStore?: (options: ResolvedMigrationCliOptions) => MigrationTargetStore;
|
||||
writeMigrationPluginConfig?: typeof writeMigrationPluginConfigDefault;
|
||||
rewriteMigrationManifest?: (params: {
|
||||
dataDir: string;
|
||||
tcvdbUrl: string;
|
||||
tcvdbDatabase: string;
|
||||
tcvdbAlias?: string;
|
||||
}) => Promise<RewriteMigrationManifestResult>;
|
||||
verifyDelayMs?: number;
|
||||
}
|
||||
|
||||
function getRequiredString(values: Record<string, string | boolean | undefined>, key: string): string {
|
||||
const value = values[key];
|
||||
if (typeof value !== "string" || value.trim() === "") {
|
||||
throw new Error(`Missing required option --${key}`);
|
||||
}
|
||||
return value.trim();
|
||||
}
|
||||
|
||||
function getOptionalString(values: Record<string, string | boolean | undefined>, key: string): string | undefined {
|
||||
const value = values[key];
|
||||
if (typeof value !== "string") return undefined;
|
||||
const trimmed = value.trim();
|
||||
return trimmed === "" ? undefined : trimmed;
|
||||
}
|
||||
|
||||
function resolveBooleanOption(
|
||||
values: Record<string, string | boolean | undefined>,
|
||||
key: string,
|
||||
defaultValue: boolean,
|
||||
): boolean {
|
||||
const value = values[key];
|
||||
return typeof value === "boolean" ? value : defaultValue;
|
||||
}
|
||||
|
||||
function resolveTcvdbApiKey(values: Record<string, string | boolean | undefined>): string {
|
||||
const directApiKey = getOptionalString(values, "tcvdb-api-key");
|
||||
const apiKeyEnvName = getOptionalString(values, "tcvdb-api-key-env");
|
||||
|
||||
if (directApiKey && apiKeyEnvName) {
|
||||
throw new Error("Provide either --tcvdb-api-key or --tcvdb-api-key-env, not both");
|
||||
}
|
||||
if (directApiKey) {
|
||||
return directApiKey;
|
||||
}
|
||||
if (!apiKeyEnvName) {
|
||||
throw new Error("Missing required TCVDB API key input: use --tcvdb-api-key or --tcvdb-api-key-env");
|
||||
}
|
||||
|
||||
const envValue = process.env[apiKeyEnvName]?.trim();
|
||||
if (!envValue) {
|
||||
throw new Error(`Environment variable ${apiKeyEnvName} is empty or not set`);
|
||||
}
|
||||
return envValue;
|
||||
}
|
||||
|
||||
function parseLayers(rawLayers: string | undefined): MigrationLayer[] {
|
||||
if (!rawLayers) return [...ALL_MIGRATION_LAYERS];
|
||||
|
||||
const values = rawLayers
|
||||
.split(",")
|
||||
.map((layer) => layer.trim())
|
||||
.filter(Boolean);
|
||||
|
||||
if (values.length === 0) {
|
||||
throw new Error("--layers must include at least one layer");
|
||||
}
|
||||
|
||||
const uniqueLayers = [...new Set(values)];
|
||||
const invalid = uniqueLayers.filter(
|
||||
(layer): layer is string => !ALL_MIGRATION_LAYERS.includes(layer as MigrationLayer),
|
||||
);
|
||||
if (invalid.length > 0) {
|
||||
throw new Error(`Unsupported layer(s): ${invalid.join(", ")}`);
|
||||
}
|
||||
|
||||
return uniqueLayers as MigrationLayer[];
|
||||
}
|
||||
|
||||
function parseBm25Language(rawLanguage: string | undefined): Bm25Language {
|
||||
if (!rawLanguage) return "zh";
|
||||
if (rawLanguage === "zh" || rawLanguage === "en") {
|
||||
return rawLanguage;
|
||||
}
|
||||
throw new Error(`Unsupported --bm25-language value: ${rawLanguage}`);
|
||||
}
|
||||
|
||||
function parseTimeout(rawValue: string | undefined): number {
|
||||
if (!rawValue) return 10000;
|
||||
const timeout = Number(rawValue);
|
||||
if (!Number.isFinite(timeout) || timeout <= 0) {
|
||||
throw new Error(`Invalid --tcvdb-timeout-ms value: ${rawValue}`);
|
||||
}
|
||||
return timeout;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a "nothing to migrate" summary — used when source data dir or sqlite
|
||||
* file doesn't exist (e.g. fresh deployment that hasn't captured any data yet).
|
||||
*/
|
||||
function buildEmptySummary(options: ResolvedMigrationCliOptions): MigrationPreflightSummary {
|
||||
return {
|
||||
pluginId: options.pluginId,
|
||||
dryRun: options.dryRun,
|
||||
layers: options.layers,
|
||||
paths: {
|
||||
pluginDataDir: options.pluginDataDir,
|
||||
sqlitePath: options.sqlitePath,
|
||||
openclawConfigPath: options.openclawConfigPath,
|
||||
},
|
||||
source: {
|
||||
l0Count: 0,
|
||||
l1Count: 0,
|
||||
profileCount: 0,
|
||||
manifestExists: false,
|
||||
manifestStoreType: null,
|
||||
},
|
||||
target: {
|
||||
url: options.tcvdb.url,
|
||||
username: options.tcvdb.username,
|
||||
database: options.tcvdb.database,
|
||||
alias: options.tcvdb.alias,
|
||||
embeddingModel: options.tcvdb.embeddingModel,
|
||||
timeout: options.tcvdb.timeout,
|
||||
bm25Enabled: options.bm25Enabled,
|
||||
bm25Language: options.bm25Language,
|
||||
},
|
||||
options: {
|
||||
applyConfig: options.applyConfig,
|
||||
configBackup: options.configBackup,
|
||||
rewriteManifest: options.rewriteManifest,
|
||||
failIfTargetNonempty: options.failIfTargetNonempty,
|
||||
verifyCounts: options.verifyCounts,
|
||||
yes: options.yes,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function ensureReadablePath(filePath: string, label: string): Promise<void> {
|
||||
try {
|
||||
await fs.access(filePath);
|
||||
} catch {
|
||||
throw new Error(`${label} does not exist or is not accessible: ${filePath}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function ensureReadableDirectory(dirPath: string, label: string): Promise<void> {
|
||||
const stat = await fs.stat(dirPath).catch(() => null);
|
||||
if (!stat?.isDirectory()) {
|
||||
throw new Error(`${label} is not a directory: ${dirPath}`);
|
||||
}
|
||||
}
|
||||
|
||||
function safeParseMetadata(raw: string): Record<string, unknown> {
|
||||
if (!raw) return {};
|
||||
try {
|
||||
const parsed = JSON.parse(raw);
|
||||
return parsed && typeof parsed === "object" && !Array.isArray(parsed)
|
||||
? (parsed as Record<string, unknown>)
|
||||
: {};
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
function compactTimestamps(row: L1RecordRow): string[] {
|
||||
return [...new Set([row.timestamp_str, row.timestamp_start, row.timestamp_end].filter(Boolean))];
|
||||
}
|
||||
|
||||
function mapL1RowToMemoryRecord(row: L1RecordRow): MemoryRecord {
|
||||
const timestamps = compactTimestamps(row);
|
||||
const fallbackIso = row.updated_time || row.created_time || row.timestamp_end || row.timestamp_str || new Date(0).toISOString();
|
||||
return {
|
||||
id: row.record_id,
|
||||
content: row.content,
|
||||
type: row.type as MemoryRecord["type"],
|
||||
priority: row.priority,
|
||||
scene_name: row.scene_name,
|
||||
source_message_ids: [],
|
||||
metadata: safeParseMetadata(row.metadata_json),
|
||||
timestamps,
|
||||
createdAt: row.created_time || fallbackIso,
|
||||
updatedAt: row.updated_time || row.created_time || fallbackIso,
|
||||
sessionKey: row.session_key || "",
|
||||
sessionId: row.session_id || "",
|
||||
};
|
||||
}
|
||||
|
||||
function mapL0RowToRecord(row: L0RecordRow): L0Record {
|
||||
return {
|
||||
id: row.record_id,
|
||||
sessionKey: row.session_key,
|
||||
sessionId: row.session_id || "",
|
||||
role: row.role,
|
||||
messageText: row.message_text,
|
||||
recordedAt: row.recorded_at || "",
|
||||
timestamp: row.timestamp ?? 0,
|
||||
};
|
||||
}
|
||||
|
||||
function createTargetStoreDefault(options: ResolvedMigrationCliOptions): MigrationTargetStore {
|
||||
const bm25Encoder = createBM25Encoder(
|
||||
{
|
||||
enabled: options.bm25Enabled,
|
||||
language: options.bm25Language,
|
||||
},
|
||||
);
|
||||
|
||||
return new TcvdbMemoryStore({
|
||||
url: options.tcvdb.url,
|
||||
username: options.tcvdb.username,
|
||||
apiKey: options.tcvdb.apiKey,
|
||||
database: options.tcvdb.database,
|
||||
embeddingModel: options.tcvdb.embeddingModel,
|
||||
timeout: options.tcvdb.timeout,
|
||||
caPemPath: options.tcvdb.caPemPath,
|
||||
bm25Encoder,
|
||||
});
|
||||
}
|
||||
|
||||
async function ensureTargetIsEmpty(
|
||||
options: ResolvedMigrationCliOptions,
|
||||
targetStore: MigrationTargetStore,
|
||||
): Promise<void> {
|
||||
if (!options.failIfTargetNonempty) return;
|
||||
|
||||
const [existingL1, existingL0, existingProfiles] = await Promise.all([
|
||||
Promise.resolve(targetStore.countL1()),
|
||||
Promise.resolve(targetStore.countL0()),
|
||||
targetStore.pullProfiles ? targetStore.pullProfiles().then((records) => records.length) : Promise.resolve(0),
|
||||
]);
|
||||
|
||||
if (existingL1 > 0 || existingL0 > 0 || existingProfiles > 0) {
|
||||
throw new Error(
|
||||
`Target store is not empty (L1=${existingL1}, L0=${existingL0}, profiles=${existingProfiles})`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function migrateL1Records(sourceStore: VectorStore, targetStore: MigrationTargetStore, pageSize: number): Promise<number> {
|
||||
let migrated = 0;
|
||||
let cursor = "";
|
||||
const useBatch = typeof targetStore.upsertL1Batch === "function";
|
||||
|
||||
// eslint-disable-next-line no-constant-condition
|
||||
while (true) {
|
||||
const rows = sourceStore.queryL1RecordsCursor(cursor, pageSize);
|
||||
if (rows.length === 0) break;
|
||||
|
||||
const records = rows.map(mapL1RowToMemoryRecord);
|
||||
|
||||
if (useBatch) {
|
||||
const count = await targetStore.upsertL1Batch!(records);
|
||||
if (count === 0) {
|
||||
throw new Error(`Failed to batch migrate L1 records (cursor=${cursor}, page=${rows.length})`);
|
||||
}
|
||||
} else {
|
||||
for (const record of records) {
|
||||
const ok = await Promise.resolve(targetStore.upsertL1(record));
|
||||
if (!ok) throw new Error(`Failed to migrate L1 record ${record.id}`);
|
||||
}
|
||||
}
|
||||
|
||||
migrated += rows.length;
|
||||
cursor = rows[rows.length - 1].record_id;
|
||||
log(`L1: 已迁移 ${migrated} 条...`);
|
||||
|
||||
if (rows.length < pageSize) break;
|
||||
}
|
||||
|
||||
log(`L1: 迁移完成，共 ${migrated} 条`);
|
||||
return migrated;
|
||||
}
|
||||
|
||||
async function migrateL0Records(sourceStore: VectorStore, targetStore: MigrationTargetStore, pageSize: number): Promise<number> {
|
||||
let migrated = 0;
|
||||
let cursor = "";
|
||||
const useBatch = typeof targetStore.upsertL0Batch === "function";
|
||||
|
||||
// eslint-disable-next-line no-constant-condition
|
||||
while (true) {
|
||||
const rows = sourceStore.queryL0RecordsCursor(cursor, pageSize);
|
||||
if (rows.length === 0) break;
|
||||
|
||||
const records = rows.map(mapL0RowToRecord);
|
||||
|
||||
if (useBatch) {
|
||||
const count = await targetStore.upsertL0Batch!(records);
|
||||
if (count === 0) {
|
||||
throw new Error(`Failed to batch migrate L0 records (cursor=${cursor}, page=${rows.length})`);
|
||||
}
|
||||
} else {
|
||||
for (const record of records) {
|
||||
const ok = await Promise.resolve(targetStore.upsertL0(record));
|
||||
if (!ok) throw new Error(`Failed to migrate L0 record ${record.id}`);
|
||||
}
|
||||
}
|
||||
|
||||
migrated += rows.length;
|
||||
cursor = rows[rows.length - 1].record_id;
|
||||
log(`L0: 已迁移 ${migrated} 条...`);
|
||||
|
||||
if (rows.length < pageSize) break;
|
||||
}
|
||||
|
||||
log(`L0: 迁移完成，共 ${migrated} 条`);
|
||||
return migrated;
|
||||
}
|
||||
|
||||
async function migrateProfiles(
|
||||
pluginDataDir: string,
|
||||
targetStore: MigrationTargetStore,
|
||||
): Promise<number> {
|
||||
const profiles = await listLocalProfiles(pluginDataDir);
|
||||
if (profiles.length === 0) {
|
||||
log("Profiles: 无本地 profile，跳过");
|
||||
return 0;
|
||||
}
|
||||
if (!targetStore.syncProfiles) {
|
||||
throw new Error("Target store does not support profile sync");
|
||||
}
|
||||
log(`Profiles: 发现 ${profiles.length} 个本地 profile，开始同步...`);
|
||||
await targetStore.syncProfiles(profiles);
|
||||
log(`Profiles: 同步完成，共 ${profiles.length} 个`);
|
||||
return profiles.length;
|
||||
}
|
||||
|
||||
async function verifyMigratedCounts(
|
||||
summary: MigrationPreflightSummary,
|
||||
targetStore: MigrationTargetStore,
|
||||
delayMs: number,
|
||||
): Promise<{ l1Count: number; l0Count: number; profileCount: number }> {
|
||||
if (delayMs > 0) {
|
||||
log(`等待 ${Math.round(delayMs / 1000)} 秒让远端数据落盘...`);
|
||||
await sleep(delayMs);
|
||||
}
|
||||
log("开始校验迁移数量...");
|
||||
|
||||
const [l1Count, l0Count, profileCount] = await Promise.all([
|
||||
Promise.resolve(targetStore.countL1()),
|
||||
Promise.resolve(targetStore.countL0()),
|
||||
targetStore.pullProfiles ? targetStore.pullProfiles().then((records) => records.length) : Promise.resolve(0),
|
||||
]);
|
||||
|
||||
log(`校验结果: L1=${l1Count}/${summary.source.l1Count}, L0=${l0Count}/${summary.source.l0Count}, Profiles=${profileCount}/${summary.source.profileCount}`);
|
||||
|
||||
if (l1Count !== summary.source.l1Count) {
|
||||
throw new Error(`L1 count verification failed: source=${summary.source.l1Count}, target=${l1Count}`);
|
||||
}
|
||||
if (l0Count !== summary.source.l0Count) {
|
||||
throw new Error(`L0 count verification failed: source=${summary.source.l0Count}, target=${l0Count}`);
|
||||
}
|
||||
if (profileCount !== summary.source.profileCount) {
|
||||
throw new Error(
|
||||
`Profile count verification failed: source=${summary.source.profileCount}, target=${profileCount}`,
|
||||
);
|
||||
}
|
||||
|
||||
log("校验通过");
|
||||
return { l1Count, l0Count, profileCount };
|
||||
}
|
||||
|
||||
async function writeSummaryJson(
|
||||
summaryJsonPath: string | undefined,
|
||||
summary: MigrationPreflightSummary,
|
||||
): Promise<void> {
|
||||
if (!summaryJsonPath) return;
|
||||
await fs.mkdir(path.dirname(summaryJsonPath), { recursive: true });
|
||||
await fs.writeFile(summaryJsonPath, `${JSON.stringify(summary, null, 2)}\n`, "utf-8");
|
||||
}
|
||||
|
||||
function printUsageAndExit(): never {
|
||||
const text = `
|
||||
SQLite → 腾讯云向量数据库迁移工具
|
||||
|
||||
Usage:
|
||||
migrate-sqlite-to-tcvdb [options]
|
||||
|
||||
Required:
|
||||
--plugin-data-dir <path> 插件数据目录路径
|
||||
--openclaw-config-path <path> openclaw.json 配置文件路径
|
||||
--tcvdb-url <url> TCVDB 服务地址
|
||||
--tcvdb-username <name> TCVDB 用户名
|
||||
--tcvdb-database <name> TCVDB 数据库名
|
||||
--tcvdb-embedding-model <name> Embedding 模型名称
|
||||
--tcvdb-api-key <key> TCVDB API 密钥（明文，与 --tcvdb-api-key-env 二选一）
|
||||
--tcvdb-api-key-env <var> 包含 API 密钥的环境变量名
|
||||
|
||||
Optional:
|
||||
--sqlite-path <path> SQLite 数据库路径（默认: <plugin-data-dir>/vectors.db）
|
||||
--plugin-id <id> 写入配置时使用的插件 ID（默认: memory-tencentdb）
|
||||
--layers <l0,l1,l2,l3> 要迁移的层，逗号分隔（默认: l0,l1,l2,l3）
|
||||
--tcvdb-alias <name> 用户自定义别名
|
||||
--tcvdb-timeout-ms <ms> 请求超时（默认: 10000）
|
||||
--tcvdb-ca-pem <path> CA 证书 PEM 文件路径（HTTPS 连接时使用）
|
||||
--bm25-language <zh|en> BM25 分词语言（默认: zh）
|
||||
--summary-json-path <path> 将迁移摘要写入此文件
|
||||
--job-id <id> 迁移任务 ID（用于追踪）
|
||||
|
||||
Flags:
|
||||
--dry-run 仅预览，不执行写入
|
||||
--yes 跳过交互确认
|
||||
--no-apply-config 不自动更新 openclaw.json
|
||||
--no-config-backup 写入配置前不备份
|
||||
--no-rewrite-manifest 不更新 manifest.json
|
||||
--no-fail-if-target-nonempty 目标库非空时不中止
|
||||
--no-verify-counts 迁移后不校验记录数
|
||||
--no-bm25-enabled 禁用 BM25 稀疏向量
|
||||
|
||||
-h, --help 显示此帮助信息
|
||||
|
||||
Examples:
|
||||
# 预检模式
|
||||
migrate-sqlite-to-tcvdb \\
|
||||
--plugin-data-dir ~/.openclaw/memory-tdai \\
|
||||
--openclaw-config-path ~/.openclaw/openclaw.json \\
|
||||
--tcvdb-url http://127.0.0.1:80 --tcvdb-username root \\
|
||||
--tcvdb-api-key-env TCVDB_API_KEY \\
|
||||
--tcvdb-database agent_memory_prod \\
|
||||
--tcvdb-embedding-model bge-large-zh \\
|
||||
--dry-run
|
||||
|
||||
# 正式迁移
|
||||
migrate-sqlite-to-tcvdb \\
|
||||
--plugin-data-dir ~/.openclaw/memory-tdai \\
|
||||
--openclaw-config-path ~/.openclaw/openclaw.json \\
|
||||
--tcvdb-url http://127.0.0.1:80 --tcvdb-username root \\
|
||||
--tcvdb-api-key-env TCVDB_API_KEY \\
|
||||
--tcvdb-database agent_memory_prod \\
|
||||
--tcvdb-embedding-model bge-large-zh \\
|
||||
--yes
|
||||
`.trimStart();
|
||||
process.stdout.write(text);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
export function resolveMigrationCliOptions(argv: string[]): ResolvedMigrationCliOptions {
|
||||
const { values } = parseArgs({
|
||||
args: argv,
|
||||
strict: true,
|
||||
allowPositionals: false,
|
||||
allowNegative: true,
|
||||
options: {
|
||||
help: { type: "boolean", short: "h" },
|
||||
"plugin-data-dir": { type: "string" },
|
||||
"sqlite-path": { type: "string" },
|
||||
"openclaw-config-path": { type: "string" },
|
||||
"plugin-id": { type: "string" },
|
||||
layers: { type: "string" },
|
||||
"apply-config": { type: "boolean" },
|
||||
"config-backup": { type: "boolean" },
|
||||
"rewrite-manifest": { type: "boolean" },
|
||||
"fail-if-target-nonempty": { type: "boolean" },
|
||||
"verify-counts": { type: "boolean" },
|
||||
"dry-run": { type: "boolean" },
|
||||
yes: { type: "boolean" },
|
||||
"bm25-enabled": { type: "boolean" },
|
||||
"bm25-language": { type: "string" },
|
||||
"summary-json-path": { type: "string" },
|
||||
"job-id": { type: "string" },
|
||||
"tcvdb-url": { type: "string" },
|
||||
"tcvdb-username": { type: "string" },
|
||||
"tcvdb-api-key": { type: "string" },
|
||||
"tcvdb-api-key-env": { type: "string" },
|
||||
"tcvdb-database": { type: "string" },
|
||||
"tcvdb-alias": { type: "string" },
|
||||
"tcvdb-embedding-model": { type: "string" },
|
||||
"tcvdb-timeout-ms": { type: "string" },
|
||||
"tcvdb-ca-pem": { type: "string" },
|
||||
},
|
||||
});
|
||||
|
||||
if (values.help) {
|
||||
printUsageAndExit();
|
||||
}
|
||||
|
||||
const pluginDataDir = path.resolve(getRequiredString(values, "plugin-data-dir"));
|
||||
const sqlitePath = path.resolve(
|
||||
getOptionalString(values, "sqlite-path") ?? path.join(pluginDataDir, "vectors.db"),
|
||||
);
|
||||
const summaryJsonPath = getOptionalString(values, "summary-json-path");
|
||||
|
||||
return {
|
||||
pluginDataDir,
|
||||
sqlitePath,
|
||||
openclawConfigPath: path.resolve(getRequiredString(values, "openclaw-config-path")),
|
||||
pluginId: getOptionalString(values, "plugin-id") ?? DEFAULT_MIGRATION_PLUGIN_ID,
|
||||
layers: parseLayers(getOptionalString(values, "layers")),
|
||||
applyConfig: resolveBooleanOption(values, "apply-config", true),
|
||||
configBackup: resolveBooleanOption(values, "config-backup", true),
|
||||
rewriteManifest: resolveBooleanOption(values, "rewrite-manifest", true),
|
||||
failIfTargetNonempty: resolveBooleanOption(values, "fail-if-target-nonempty", true),
|
||||
verifyCounts: resolveBooleanOption(values, "verify-counts", true),
|
||||
dryRun: resolveBooleanOption(values, "dry-run", false),
|
||||
yes: resolveBooleanOption(values, "yes", false),
|
||||
bm25Enabled: resolveBooleanOption(values, "bm25-enabled", true),
|
||||
bm25Language: parseBm25Language(getOptionalString(values, "bm25-language")),
|
||||
summaryJsonPath: summaryJsonPath ? path.resolve(summaryJsonPath) : undefined,
|
||||
jobId: getOptionalString(values, "job-id"),
|
||||
tcvdb: {
|
||||
url: getRequiredString(values, "tcvdb-url"),
|
||||
username: getRequiredString(values, "tcvdb-username"),
|
||||
apiKey: resolveTcvdbApiKey(values),
|
||||
database: getRequiredString(values, "tcvdb-database"),
|
||||
alias: getOptionalString(values, "tcvdb-alias") ?? "",
|
||||
embeddingModel: getRequiredString(values, "tcvdb-embedding-model"),
|
||||
timeout: parseTimeout(getOptionalString(values, "tcvdb-timeout-ms")),
|
||||
caPemPath: getOptionalString(values, "tcvdb-ca-pem"),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export async function collectMigrationPreflight(
|
||||
options: ResolvedMigrationCliOptions,
|
||||
): Promise<MigrationPreflightSummary> {
|
||||
// ── 优雅处理"数据目录 / sqlite 不存在"的场景 ──
|
||||
// 如果源数据路径不存在（全新部署、尚未有任何 capture），不报错——
|
||||
// 返回"全零"summary，让调用方判断是否跳过迁移。
|
||||
const dirExists = await fs.stat(options.pluginDataDir).then(s => s.isDirectory()).catch(() => false);
|
||||
if (!dirExists) {
|
||||
log(`plugin data directory 不存在，无需迁移: ${options.pluginDataDir}`);
|
||||
return buildEmptySummary(options);
|
||||
}
|
||||
const sqliteExists = await fs.access(options.sqlitePath).then(() => true).catch(() => false);
|
||||
if (!sqliteExists) {
|
||||
log(`sqlite database 不存在，无需迁移: ${options.sqlitePath}`);
|
||||
return buildEmptySummary(options);
|
||||
}
|
||||
await ensureReadablePath(options.openclawConfigPath, "OpenClaw config file");
|
||||
|
||||
const store = new VectorStore(options.sqlitePath, 0);
|
||||
const initResult = store.init();
|
||||
if (store.isDegraded()) {
|
||||
store.close();
|
||||
throw new Error(`Failed to open sqlite store for migration preflight: ${initResult.reason ?? "unknown error"}`);
|
||||
}
|
||||
|
||||
try {
|
||||
const [profiles, manifest] = await Promise.all([
|
||||
listLocalProfiles(options.pluginDataDir),
|
||||
Promise.resolve(readManifest(options.pluginDataDir)),
|
||||
]);
|
||||
|
||||
return {
|
||||
pluginId: options.pluginId,
|
||||
dryRun: options.dryRun,
|
||||
layers: options.layers,
|
||||
paths: {
|
||||
pluginDataDir: options.pluginDataDir,
|
||||
sqlitePath: options.sqlitePath,
|
||||
openclawConfigPath: options.openclawConfigPath,
|
||||
},
|
||||
source: {
|
||||
l0Count: store.countL0(),
|
||||
l1Count: store.countL1(),
|
||||
profileCount: profiles.length,
|
||||
manifestExists: manifest !== null,
|
||||
manifestStoreType: manifest?.store.type ?? null,
|
||||
},
|
||||
target: {
|
||||
url: options.tcvdb.url,
|
||||
username: options.tcvdb.username,
|
||||
database: options.tcvdb.database,
|
||||
alias: options.tcvdb.alias,
|
||||
embeddingModel: options.tcvdb.embeddingModel,
|
||||
timeout: options.tcvdb.timeout,
|
||||
bm25Enabled: options.bm25Enabled,
|
||||
bm25Language: options.bm25Language,
|
||||
},
|
||||
options: {
|
||||
applyConfig: options.applyConfig,
|
||||
configBackup: options.configBackup,
|
||||
rewriteManifest: options.rewriteManifest,
|
||||
failIfTargetNonempty: options.failIfTargetNonempty,
|
||||
verifyCounts: options.verifyCounts,
|
||||
yes: options.yes,
|
||||
},
|
||||
};
|
||||
} finally {
|
||||
store.close();
|
||||
}
|
||||
}
|
||||
|
||||
export async function runMigrationCli(
|
||||
argv: string[],
|
||||
deps: RunMigrationCliDeps = {},
|
||||
): Promise<MigrationPreflightSummary> {
|
||||
const options = resolveMigrationCliOptions(argv);
|
||||
log("开始预检...");
|
||||
const summary = await collectMigrationPreflight(options);
|
||||
log(`预检完成: 源数据 L1=${summary.source.l1Count}, L0=${summary.source.l0Count}, Profiles=${summary.source.profileCount}`);
|
||||
log(`目标: ${summary.target.url} / ${summary.target.database}`);
|
||||
|
||||
const hasSourceData = summary.source.l0Count > 0 || summary.source.l1Count > 0 || summary.source.profileCount > 0;
|
||||
|
||||
if (!hasSourceData) {
|
||||
log("源数据为空，跳过数据迁移。");
|
||||
}
|
||||
|
||||
if (options.dryRun) {
|
||||
log("预检模式 (dry-run)，不执行写入");
|
||||
await writeSummaryJson(options.summaryJsonPath, summary);
|
||||
return summary;
|
||||
}
|
||||
|
||||
const createTargetStore = deps.createTargetStore ?? createTargetStoreDefault;
|
||||
const writeMigrationPluginConfig = deps.writeMigrationPluginConfig ?? writeMigrationPluginConfigDefault;
|
||||
const rewriteMigrationManifest = deps.rewriteMigrationManifest ?? (async (params) =>
|
||||
await rewriteMigrationManifestDefault(params));
|
||||
|
||||
const migration = {
|
||||
l1Migrated: 0,
|
||||
l0Migrated: 0,
|
||||
profileMigrated: 0,
|
||||
targetL1Count: 0,
|
||||
targetL0Count: 0,
|
||||
targetProfileCount: 0,
|
||||
configWritten: false,
|
||||
manifestWritten: false,
|
||||
manifestBackupPath: undefined as string | undefined,
|
||||
};
|
||||
|
||||
if (hasSourceData) {
|
||||
log("打开 SQLite 源库...");
|
||||
const sourceStore = new VectorStore(options.sqlitePath, 0);
|
||||
const sourceInitResult = sourceStore.init();
|
||||
if (sourceStore.isDegraded()) {
|
||||
sourceStore.close();
|
||||
throw new Error(`Failed to reopen sqlite source store: ${sourceInitResult.reason ?? "unknown error"}`);
|
||||
}
|
||||
|
||||
log("初始化目标库...");
|
||||
const targetStore = createTargetStore(options);
|
||||
|
||||
try {
|
||||
await Promise.resolve(targetStore.init());
|
||||
if (targetStore.isDegraded()) {
|
||||
throw new Error("Target store entered degraded mode during initialization");
|
||||
}
|
||||
|
||||
await ensureTargetIsEmpty(options, targetStore);
|
||||
|
||||
const pageSize = DEFAULT_MIGRATION_PAGE_SIZE;
|
||||
log(`分页大小: ${pageSize} 条/批`);
|
||||
|
||||
migration.l1Migrated = options.layers.includes("l1") ? await migrateL1Records(sourceStore, targetStore, pageSize) : 0;
|
||||
migration.l0Migrated = options.layers.includes("l0") ? await migrateL0Records(sourceStore, targetStore, pageSize) : 0;
|
||||
migration.profileMigrated = options.layers.includes("l2") || options.layers.includes("l3")
|
||||
? await migrateProfiles(options.pluginDataDir, targetStore)
|
||||
: 0;
|
||||
|
||||
const verifyDelayMs = deps.verifyDelayMs ?? 10_000;
|
||||
|
||||
const verifiedCounts = options.verifyCounts
|
||||
? await verifyMigratedCounts(summary, targetStore, verifyDelayMs)
|
||||
: {
|
||||
l1Count: await Promise.resolve(targetStore.countL1()),
|
||||
l0Count: await Promise.resolve(targetStore.countL0()),
|
||||
profileCount: targetStore.pullProfiles ? (await targetStore.pullProfiles()).length : 0,
|
||||
};
|
||||
|
||||
migration.targetL1Count = verifiedCounts.l1Count;
|
||||
migration.targetL0Count = verifiedCounts.l0Count;
|
||||
migration.targetProfileCount = verifiedCounts.profileCount;
|
||||
} finally {
|
||||
sourceStore.close();
|
||||
targetStore.close();
|
||||
}
|
||||
}
|
||||
|
||||
if (options.applyConfig) {
|
||||
log(`写入配置到 ${options.openclawConfigPath} ...`);
|
||||
await writeMigrationPluginConfig({
|
||||
configPath: options.openclawConfigPath,
|
||||
pluginId: options.pluginId,
|
||||
tcvdb: {
|
||||
url: options.tcvdb.url,
|
||||
username: options.tcvdb.username,
|
||||
apiKey: options.tcvdb.apiKey,
|
||||
database: options.tcvdb.database,
|
||||
alias: options.tcvdb.alias,
|
||||
embeddingModel: options.tcvdb.embeddingModel,
|
||||
timeout: options.tcvdb.timeout,
|
||||
},
|
||||
bm25: {
|
||||
enabled: options.bm25Enabled,
|
||||
language: options.bm25Language,
|
||||
},
|
||||
});
|
||||
migration.configWritten = true;
|
||||
log("配置写入完成");
|
||||
}
|
||||
|
||||
if (options.rewriteManifest) {
|
||||
log("更新 manifest...");
|
||||
const manifestResult = await rewriteMigrationManifest({
|
||||
dataDir: options.pluginDataDir,
|
||||
tcvdbUrl: options.tcvdb.url,
|
||||
tcvdbDatabase: options.tcvdb.database,
|
||||
tcvdbAlias: options.tcvdb.alias || undefined,
|
||||
});
|
||||
migration.manifestWritten = manifestResult.created || manifestResult.updated;
|
||||
migration.manifestBackupPath = manifestResult.backupPath;
|
||||
log(`Manifest ${manifestResult.created ? "创建" : "更新"}完成${manifestResult.backupPath ? `，备份: ${manifestResult.backupPath}` : ""}`);
|
||||
}
|
||||
|
||||
summary.migration = {
|
||||
l1Migrated: migration.l1Migrated,
|
||||
l0Migrated: migration.l0Migrated,
|
||||
profileMigrated: migration.profileMigrated,
|
||||
targetL1Count: migration.targetL1Count,
|
||||
targetL0Count: migration.targetL0Count,
|
||||
targetProfileCount: migration.targetProfileCount,
|
||||
configWritten: migration.configWritten,
|
||||
manifestWritten: migration.manifestWritten,
|
||||
manifestBackupPath: migration.manifestBackupPath,
|
||||
};
|
||||
|
||||
await writeSummaryJson(options.summaryJsonPath, summary);
|
||||
log("迁移全部完成!");
|
||||
return summary;
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "es2023",
|
||||
"module": "NodeNext",
|
||||
"moduleResolution": "NodeNext",
|
||||
"declaration": false,
|
||||
"strict": true,
|
||||
"esModuleInterop": true,
|
||||
"skipLibCheck": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"resolveJsonModule": true,
|
||||
"types": ["node"],
|
||||
"paths": {
|
||||
"node-llama-cpp": ["./node-llama-cpp.d.ts"]
|
||||
},
|
||||
"outDir": "dist",
|
||||
"rootDir": "../.."
|
||||
},
|
||||
"files": [
|
||||
"cli-entry.ts",
|
||||
"sqlite-to-tcvdb.ts",
|
||||
"config-write.ts",
|
||||
"manifest-write.ts"
|
||||
]
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2022",
|
||||
"module": "Node16",
|
||||
"moduleResolution": "Node16",
|
||||
"outDir": "./dist",
|
||||
"rootDir": ".",
|
||||
"strict": true,
|
||||
"esModuleInterop": true,
|
||||
"skipLibCheck": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"resolveJsonModule": true,
|
||||
"types": ["node"],
|
||||
"declaration": false,
|
||||
"sourceMap": false
|
||||
},
|
||||
"include": ["read-local-memory.ts"],
|
||||
"exclude": ["dist", "node_modules"]
|
||||
}
|
||||
Reference in New Issue
Block a user