mirror of
https://github.com/TencentCloud/TencentDB-Agent-Memory
synced 2026-07-10 12:34:27 +00:00
chore: release v0.3.5
This commit is contained in:
chrishuan
@@ -0,0 +1,239 @@
|
||||
# SQLite → 腾讯云向量数据库迁移工具
|
||||
|
||||
离线迁移工具，用于将 memory-tdai 的数据从本地 SQLite 存储迁移到腾讯云向量数据库（TCVDB）。
|
||||
|
||||
## 前置条件
|
||||
|
||||
- Node.js >= 22.16.0
|
||||
- 插件已通过 `openclaw plugins install` 安装
|
||||
- 迁移脚本已编译（见下方）
|
||||
|
||||
## 编译
|
||||
|
||||
迁移脚本使用 TypeScript 编写，运行前需要先编译：
|
||||
|
||||
```bash
|
||||
npm run build:migrate-sqlite-to-vdb
|
||||
```
|
||||
|
||||
编译产物输出到 `scripts/migrate-sqlite-to-tcvdb/dist/`，可直接用 Node 运行。
|
||||
|
||||
## 使用方法
|
||||
|
||||
```bash
|
||||
# 预检模式（仅查看源数据，不执行写入）
|
||||
npm run migrate:sqlite-to-tcvdb -- \
|
||||
--plugin-data-dir ~/.openclaw/memory-tdai \
|
||||
--openclaw-config-path ~/.openclaw/openclaw.json \
|
||||
--tcvdb-url http://127.0.0.1:80 \
|
||||
--tcvdb-username root \
|
||||
--tcvdb-api-key-env TCVDB_API_KEY \
|
||||
--tcvdb-database agent_memory_prod \
|
||||
--tcvdb-embedding-model bge-large-zh \
|
||||
--dry-run
|
||||
|
||||
# 正式迁移
|
||||
npm run migrate:sqlite-to-tcvdb -- \
|
||||
--plugin-data-dir ~/.openclaw/memory-tdai \
|
||||
--openclaw-config-path ~/.openclaw/openclaw.json \
|
||||
--tcvdb-url http://127.0.0.1:80 \
|
||||
--tcvdb-username root \
|
||||
--tcvdb-api-key-env TCVDB_API_KEY \
|
||||
--tcvdb-database agent_memory_prod \
|
||||
--tcvdb-embedding-model bge-large-zh \
|
||||
--yes
|
||||
```
|
||||
|
||||
### 更多示例
|
||||
|
||||
```bash
|
||||
# 直接传入 API Key（不通过环境变量）
|
||||
npm run migrate:sqlite-to-tcvdb -- \
|
||||
--plugin-data-dir ~/.openclaw/memory-tdai \
|
||||
--openclaw-config-path ~/.openclaw/openclaw.json \
|
||||
--tcvdb-url http://127.0.0.1:80 \
|
||||
--tcvdb-username root \
|
||||
--tcvdb-api-key 'your-api-key-here' \
|
||||
--tcvdb-database agent_memory_prod \
|
||||
--tcvdb-embedding-model bge-large-zh \
|
||||
--yes
|
||||
```
|
||||
|
||||
```bash
|
||||
# 指定自定义 SQLite 路径（数据库不在默认 vectors.db 位置时）
|
||||
npm run migrate:sqlite-to-tcvdb -- \
|
||||
--plugin-data-dir ~/.openclaw/memory-tdai \
|
||||
--sqlite-path /backup/2026-04/vectors-snapshot.db \
|
||||
--openclaw-config-path ~/.openclaw/openclaw.json \
|
||||
--tcvdb-url http://127.0.0.1:80 \
|
||||
--tcvdb-username root \
|
||||
--tcvdb-api-key-env TCVDB_API_KEY \
|
||||
--tcvdb-database agent_memory_prod \
|
||||
--tcvdb-embedding-model bge-large-zh \
|
||||
--yes
|
||||
```
|
||||
|
||||
```bash
|
||||
# 只迁移 L1 记忆层（跳过 L0 原始消息和 Profile）
|
||||
npm run migrate:sqlite-to-tcvdb -- \
|
||||
--plugin-data-dir ~/.openclaw/memory-tdai \
|
||||
--openclaw-config-path ~/.openclaw/openclaw.json \
|
||||
--tcvdb-url http://127.0.0.1:80 \
|
||||
--tcvdb-username root \
|
||||
--tcvdb-api-key-env TCVDB_API_KEY \
|
||||
--tcvdb-database agent_memory_prod \
|
||||
--tcvdb-embedding-model bge-large-zh \
|
||||
--layers l1 \
|
||||
--yes
|
||||
```
|
||||
|
||||
```bash
|
||||
# 只迁移 L0 和 L1（不迁移 Profile）
|
||||
npm run migrate:sqlite-to-tcvdb -- \
|
||||
--plugin-data-dir ~/.openclaw/memory-tdai \
|
||||
--openclaw-config-path ~/.openclaw/openclaw.json \
|
||||
--tcvdb-url http://127.0.0.1:80 \
|
||||
--tcvdb-username root \
|
||||
--tcvdb-api-key-env TCVDB_API_KEY \
|
||||
--tcvdb-database agent_memory_prod \
|
||||
--tcvdb-embedding-model bge-large-zh \
|
||||
--layers l0,l1 \
|
||||
--yes
|
||||
```
|
||||
|
||||
```bash
|
||||
# 英文语料场景：使用英文 BM25 分词
|
||||
npm run migrate:sqlite-to-tcvdb -- \
|
||||
--plugin-data-dir ~/.openclaw/memory-tdai \
|
||||
--openclaw-config-path ~/.openclaw/openclaw.json \
|
||||
--tcvdb-url http://127.0.0.1:80 \
|
||||
--tcvdb-username root \
|
||||
--tcvdb-api-key-env TCVDB_API_KEY \
|
||||
--tcvdb-database agent_memory_prod \
|
||||
--tcvdb-embedding-model bge-large-en-v1.5 \
|
||||
--bm25-language en \
|
||||
--yes
|
||||
```
|
||||
|
||||
```bash
|
||||
# 禁用 BM25 稀疏向量（仅使用密集向量检索）
|
||||
npm run migrate:sqlite-to-tcvdb -- \
|
||||
--plugin-data-dir ~/.openclaw/memory-tdai \
|
||||
--openclaw-config-path ~/.openclaw/openclaw.json \
|
||||
--tcvdb-url http://127.0.0.1:80 \
|
||||
--tcvdb-username root \
|
||||
--tcvdb-api-key-env TCVDB_API_KEY \
|
||||
--tcvdb-database agent_memory_prod \
|
||||
--tcvdb-embedding-model bge-large-zh \
|
||||
--no-bm25-enabled \
|
||||
--yes
|
||||
```
|
||||
|
||||
```bash
|
||||
# 仅迁移数据，不自动更新 openclaw.json 和 manifest（手动管理配置）
|
||||
npm run migrate:sqlite-to-tcvdb -- \
|
||||
--plugin-data-dir ~/.openclaw/memory-tdai \
|
||||
--openclaw-config-path ~/.openclaw/openclaw.json \
|
||||
--tcvdb-url http://127.0.0.1:80 \
|
||||
--tcvdb-username root \
|
||||
--tcvdb-api-key-env TCVDB_API_KEY \
|
||||
--tcvdb-database agent_memory_prod \
|
||||
--tcvdb-embedding-model bge-large-zh \
|
||||
--no-apply-config \
|
||||
--no-rewrite-manifest \
|
||||
--yes
|
||||
```
|
||||
|
||||
```bash
|
||||
# 追加迁移：允许目标库已有数据，跳过非空检查
|
||||
npm run migrate:sqlite-to-tcvdb -- \
|
||||
--plugin-data-dir ~/.openclaw/memory-tdai \
|
||||
--openclaw-config-path ~/.openclaw/openclaw.json \
|
||||
--tcvdb-url http://127.0.0.1:80 \
|
||||
--tcvdb-username root \
|
||||
--tcvdb-api-key-env TCVDB_API_KEY \
|
||||
--tcvdb-database agent_memory_prod \
|
||||
--tcvdb-embedding-model bge-large-zh \
|
||||
--no-fail-if-target-nonempty \
|
||||
--no-verify-counts \
|
||||
--yes
|
||||
```
|
||||
|
||||
```bash
|
||||
# 输出迁移摘要到 JSON 文件（适合 CI/自动化流水线）
|
||||
npm run migrate:sqlite-to-tcvdb -- \
|
||||
--plugin-data-dir ~/.openclaw/memory-tdai \
|
||||
--openclaw-config-path ~/.openclaw/openclaw.json \
|
||||
--tcvdb-url http://127.0.0.1:80 \
|
||||
--tcvdb-username root \
|
||||
--tcvdb-api-key-env TCVDB_API_KEY \
|
||||
--tcvdb-database agent_memory_prod \
|
||||
--tcvdb-embedding-model bge-large-zh \
|
||||
--summary-json-path ./migration-report.json \
|
||||
--job-id "migrate-2026-04-13" \
|
||||
--yes
|
||||
```
|
||||
|
||||
```bash
|
||||
# 设置自定义超时和别名
|
||||
npm run migrate:sqlite-to-tcvdb -- \
|
||||
--plugin-data-dir ~/.openclaw/memory-tdai \
|
||||
--openclaw-config-path ~/.openclaw/openclaw.json \
|
||||
--tcvdb-url http://10.0.1.50:80 \
|
||||
--tcvdb-username admin \
|
||||
--tcvdb-api-key-env TCVDB_API_KEY \
|
||||
--tcvdb-database agent_memory_prod \
|
||||
--tcvdb-embedding-model bge-large-zh \
|
||||
--tcvdb-alias "生产环境-主库" \
|
||||
--tcvdb-timeout-ms 30000 \
|
||||
--yes
|
||||
```
|
||||
|
||||
## 参数说明
|
||||
|
||||
| 参数 | 必填 | 默认值 | 说明 |
|
||||
|---|---|---|---|
|
||||
| `--plugin-data-dir` | 是 | — | 插件数据目录路径 |
|
||||
| `--openclaw-config-path` | 是 | — | `openclaw.json` 配置文件路径 |
|
||||
| `--sqlite-path` | 否 | `<plugin-data-dir>/vectors.db` | SQLite 数据库文件路径（默认取数据目录下的 `vectors.db`） |
|
||||
| `--plugin-id` | 否 | `memory-tencentdb` | 写入配置时使用的插件 ID |
|
||||
| `--tcvdb-url` | 是 | — | TCVDB 服务地址 |
|
||||
| `--tcvdb-username` | 是 | — | TCVDB 用户名 |
|
||||
| `--tcvdb-api-key` | * | — | TCVDB API 密钥（明文） |
|
||||
| `--tcvdb-api-key-env` | * | — | 包含 API 密钥的环境变量名 |
|
||||
| `--tcvdb-database` | 是 | — | TCVDB 数据库名 |
|
||||
| `--tcvdb-embedding-model` | 是 | — | Embedding 模型名称 |
|
||||
| `--tcvdb-alias` | 否 | `""` | 用户自定义别名 |
|
||||
| `--tcvdb-timeout-ms` | 否 | `10000` | 请求超时时间（毫秒） |
|
||||
| `--layers` | 否 | `l0,l1,l2,l3` | 要迁移的层（逗号分隔） |
|
||||
| `--dry-run` | 否 | `false` | 仅预览，不执行写入 |
|
||||
| `--yes` | 否 | `false` | 跳过交互确认 |
|
||||
| `--apply-config` | 否 | `true` | 迁移后更新 openclaw.json |
|
||||
| `--config-backup` | 否 | `true` | 写入配置前先备份原配置文件 |
|
||||
| `--rewrite-manifest` | 否 | `true` | 将 manifest.json 更新为 tcvdb |
|
||||
| `--fail-if-target-nonempty` | 否 | `true` | 目标库非空时中止 |
|
||||
| `--verify-counts` | 否 | `true` | 迁移后校验记录数 |
|
||||
| `--summary-json-path` | 否 | — | 将迁移摘要写入此文件 |
|
||||
| `--job-id` | 否 | — | 迁移任务 ID（用于追踪） |
|
||||
| `--bm25-enabled` | 否 | `true` | 启用 BM25 稀疏向量 |
|
||||
| `--bm25-language` | 否 | `zh` | BM25 语言（`zh` 或 `en`） |
|
||||
|
||||
\* `--tcvdb-api-key` 和 `--tcvdb-api-key-env` 二选一，必须提供其中一个。
|
||||
|
||||
## 目录结构
|
||||
|
||||
```
|
||||
scripts/migrate-sqlite-to-tcvdb/
|
||||
├── cli-entry.ts # CLI 入口
|
||||
├── sqlite-to-tcvdb.ts # 迁移核心逻辑（参数解析、预检、数据迁移）
|
||||
├── config-write.ts # OpenClaw 配置更新（JSON5，自包含）
|
||||
├── manifest-write.ts # Manifest 重写
|
||||
├── *.test.ts # 就近放置的测试文件
|
||||
├── tsconfig.json # 迁移脚本编译配置
|
||||
├── dist/ # 编译产物（已 gitignore）
|
||||
└── README.md # 本文件
|
||||
|
||||
bin/migrate-sqlite-to-tcvdb.mjs # 极薄 bin 包装 → dist/
|
||||
```
|
||||
|
||||
迁移脚本通过 `../../src/` 引用存储实现（VectorStore、TcvdbMemoryStore 等），但**不依赖 `openclaw/plugin-sdk`**。配置写回直接使用 `json5`。
|
||||
@@ -0,0 +1,12 @@
|
||||
import { runMigrationCli } from "./sqlite-to-tcvdb.js";
|
||||
|
||||
const TAG = "[memory-tdai][migrate-cli]";
|
||||
|
||||
try {
|
||||
const summary = await runMigrationCli(process.argv.slice(2));
|
||||
process.stdout.write(`${JSON.stringify(summary, null, 2)}\n`);
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
process.stderr.write(`${TAG} ${message}\n`);
|
||||
process.exitCode = 1;
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
import fs from "node:fs/promises";
|
||||
import JSON5 from "json5";
|
||||
|
||||
export type Bm25Language = "zh" | "en";
|
||||
|
||||
export interface MigrationPluginConfigTarget {
|
||||
url: string;
|
||||
username: string;
|
||||
apiKey: string;
|
||||
database: string;
|
||||
alias: string;
|
||||
embeddingModel: string;
|
||||
timeout: number;
|
||||
bm25Enabled: boolean;
|
||||
bm25Language: Bm25Language;
|
||||
}
|
||||
|
||||
export interface WriteMigrationPluginConfigParams {
|
||||
configPath: string;
|
||||
pluginId: string;
|
||||
tcvdb: {
|
||||
url: string;
|
||||
username: string;
|
||||
apiKey: string;
|
||||
database: string;
|
||||
alias: string;
|
||||
embeddingModel: string;
|
||||
timeout: number;
|
||||
};
|
||||
bm25: {
|
||||
enabled: boolean;
|
||||
language: Bm25Language;
|
||||
};
|
||||
}
|
||||
|
||||
interface ConfigWriteAdapterDeps {
|
||||
fs?: Pick<typeof fs, "readFile" | "writeFile" | "mkdir">;
|
||||
parseConfig?: (raw: string) => unknown;
|
||||
}
|
||||
|
||||
function asRecord(value: unknown): Record<string, unknown> {
|
||||
return value !== null && typeof value === "object" && !Array.isArray(value)
|
||||
? { ...(value as Record<string, unknown>) }
|
||||
: {};
|
||||
}
|
||||
|
||||
export function buildMigrationPluginConfigPatch(target: MigrationPluginConfigTarget): Record<string, unknown> {
|
||||
return {
|
||||
storeBackend: "tcvdb",
|
||||
tcvdb: {
|
||||
url: target.url,
|
||||
username: target.username,
|
||||
apiKey: target.apiKey,
|
||||
database: target.database,
|
||||
alias: target.alias,
|
||||
embeddingModel: target.embeddingModel,
|
||||
timeout: target.timeout,
|
||||
},
|
||||
bm25: {
|
||||
enabled: target.bm25Enabled,
|
||||
language: target.bm25Language,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function applyPluginConfigPatch(
|
||||
sourceConfig: Record<string, unknown>,
|
||||
pluginId: string,
|
||||
patch: Record<string, unknown>,
|
||||
): Record<string, unknown> {
|
||||
const nextConfig = asRecord(sourceConfig);
|
||||
const plugins = asRecord(nextConfig.plugins);
|
||||
const entries = asRecord(plugins.entries);
|
||||
const targetEntry = asRecord(entries[pluginId]);
|
||||
const targetPluginConfig = asRecord(targetEntry.config);
|
||||
const patchTcvdb = asRecord(patch.tcvdb);
|
||||
const patchBm25 = asRecord(patch.bm25);
|
||||
|
||||
const mergedPluginConfig: Record<string, unknown> = {
|
||||
...targetPluginConfig,
|
||||
...patch,
|
||||
tcvdb: {
|
||||
...asRecord(targetPluginConfig.tcvdb),
|
||||
...patchTcvdb,
|
||||
},
|
||||
bm25: {
|
||||
...asRecord(targetPluginConfig.bm25),
|
||||
...patchBm25,
|
||||
},
|
||||
};
|
||||
|
||||
entries[pluginId] = {
|
||||
...targetEntry,
|
||||
config: mergedPluginConfig,
|
||||
};
|
||||
|
||||
plugins.entries = entries;
|
||||
nextConfig.plugins = plugins;
|
||||
return nextConfig;
|
||||
}
|
||||
|
||||
export async function writeMigrationPluginConfig(
|
||||
params: WriteMigrationPluginConfigParams,
|
||||
deps: ConfigWriteAdapterDeps = {},
|
||||
): Promise<void> {
|
||||
const fsImpl = deps.fs ?? fs;
|
||||
const parseConfig = deps.parseConfig ?? ((raw: string) => JSON5.parse(raw));
|
||||
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = parseConfig(await fsImpl.readFile(params.configPath, "utf-8"));
|
||||
} catch {
|
||||
throw new Error(
|
||||
`Config migration writer only supports single-file JSON/JSON5: ${params.configPath}`,
|
||||
);
|
||||
}
|
||||
|
||||
const nextConfig = applyPluginConfigPatch(
|
||||
asRecord(parsed),
|
||||
params.pluginId,
|
||||
buildMigrationPluginConfigPatch({
|
||||
...params.tcvdb,
|
||||
bm25Enabled: params.bm25.enabled,
|
||||
bm25Language: params.bm25.language,
|
||||
}),
|
||||
);
|
||||
|
||||
await fsImpl.writeFile(params.configPath, `${JSON.stringify(nextConfig, null, 2)}\n`, "utf-8");
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import {
|
||||
buildStoreInfo,
|
||||
manifestPath,
|
||||
readManifest,
|
||||
writeManifest,
|
||||
} from "../../src/utils/manifest.js";
|
||||
|
||||
export interface RewriteMigrationManifestParams {
|
||||
dataDir: string;
|
||||
tcvdbUrl: string;
|
||||
tcvdbDatabase: string;
|
||||
tcvdbAlias?: string;
|
||||
backupExisting?: boolean;
|
||||
now?: () => string;
|
||||
}
|
||||
|
||||
export interface RewriteMigrationManifestResult {
|
||||
created: boolean;
|
||||
updated: boolean;
|
||||
backupPath?: string;
|
||||
}
|
||||
|
||||
const DEFAULT_BACKUP_FILE = "manifest.json.migrate.bak";
|
||||
|
||||
export async function rewriteMigrationManifest(
|
||||
params: RewriteMigrationManifestParams,
|
||||
): Promise<RewriteMigrationManifestResult> {
|
||||
const existing = readManifest(params.dataDir);
|
||||
const nextStore = buildStoreInfo({
|
||||
type: "tcvdb",
|
||||
tcvdbUrl: params.tcvdbUrl,
|
||||
tcvdbDatabase: params.tcvdbDatabase,
|
||||
tcvdbAlias: params.tcvdbAlias,
|
||||
});
|
||||
|
||||
if (!existing) {
|
||||
writeManifest(params.dataDir, {
|
||||
version: 1,
|
||||
createdAt: params.now?.() ?? new Date().toISOString(),
|
||||
store: nextStore,
|
||||
seed: null,
|
||||
});
|
||||
return {
|
||||
created: true,
|
||||
updated: false,
|
||||
backupPath: undefined,
|
||||
};
|
||||
}
|
||||
|
||||
let backupPath: string | undefined;
|
||||
if (params.backupExisting !== false) {
|
||||
backupPath = path.join(path.dirname(manifestPath(params.dataDir)), DEFAULT_BACKUP_FILE);
|
||||
await fs.mkdir(path.dirname(backupPath), { recursive: true });
|
||||
await fs.copyFile(manifestPath(params.dataDir), backupPath);
|
||||
}
|
||||
|
||||
writeManifest(params.dataDir, {
|
||||
...existing,
|
||||
store: nextStore,
|
||||
});
|
||||
|
||||
return {
|
||||
created: false,
|
||||
updated: true,
|
||||
backupPath,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
/**
|
||||
* Type stub for node-llama-cpp — the migration script does not use local
|
||||
* embedding but TypeScript still resolves the module through transitive
|
||||
* imports (sqlite.ts → embedding.ts → import("node-llama-cpp")).
|
||||
*
|
||||
* This stub satisfies the compiler without requiring the actual package.
|
||||
*/
|
||||
declare module "node-llama-cpp" {
|
||||
const _: any;
|
||||
export = _;
|
||||
}
|
||||
@@ -0,0 +1,849 @@
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { parseArgs } from "node:util";
|
||||
import type { MemoryRecord } from "../../src/core/record/l1-writer.js";
|
||||
import { listLocalProfiles } from "../../src/core/profile/profile-sync.js";
|
||||
import { createBM25Encoder } from "../../src/core/store/bm25-local.js";
|
||||
import { VectorStore, type L0RecordRow, type L1RecordRow } from "../../src/core/store/sqlite.js";
|
||||
import { TcvdbMemoryStore } from "../../src/core/store/tcvdb.js";
|
||||
import type { L0Record, ProfileRecord, ProfileSyncRecord, StoreInitResult } from "../../src/core/store/types.js";
|
||||
import { readManifest } from "../../src/utils/manifest.js";
|
||||
import {
|
||||
rewriteMigrationManifest as rewriteMigrationManifestDefault,
|
||||
type RewriteMigrationManifestResult,
|
||||
} from "./manifest-write.js";
|
||||
import {
|
||||
writeMigrationPluginConfig as writeMigrationPluginConfigDefault,
|
||||
} from "./config-write.js";
|
||||
|
||||
export const DEFAULT_MIGRATION_PLUGIN_ID = "memory-tencentdb";
|
||||
export const ALL_MIGRATION_LAYERS = ["l0", "l1", "l2", "l3"] as const;
|
||||
|
||||
const TAG = "[memory-tdai][migrate]";
|
||||
|
||||
function log(message: string): void {
|
||||
process.stderr.write(`${TAG} ${message}\n`);
|
||||
}
|
||||
|
||||
function sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
export type MigrationLayer = (typeof ALL_MIGRATION_LAYERS)[number];
|
||||
export type Bm25Language = "zh" | "en";
|
||||
|
||||
export interface ResolvedMigrationCliOptions {
|
||||
pluginDataDir: string;
|
||||
sqlitePath: string;
|
||||
openclawConfigPath: string;
|
||||
pluginId: string;
|
||||
layers: MigrationLayer[];
|
||||
applyConfig: boolean;
|
||||
configBackup: boolean;
|
||||
rewriteManifest: boolean;
|
||||
failIfTargetNonempty: boolean;
|
||||
verifyCounts: boolean;
|
||||
dryRun: boolean;
|
||||
yes: boolean;
|
||||
bm25Enabled: boolean;
|
||||
bm25Language: Bm25Language;
|
||||
summaryJsonPath?: string;
|
||||
jobId?: string;
|
||||
tcvdb: {
|
||||
url: string;
|
||||
username: string;
|
||||
apiKey: string;
|
||||
database: string;
|
||||
alias: string;
|
||||
embeddingModel: string;
|
||||
timeout: number;
|
||||
caPemPath?: string;
|
||||
};
|
||||
}
|
||||
|
||||
export interface MigrationPreflightSummary {
|
||||
pluginId: string;
|
||||
dryRun: boolean;
|
||||
layers: MigrationLayer[];
|
||||
paths: {
|
||||
pluginDataDir: string;
|
||||
sqlitePath: string;
|
||||
openclawConfigPath: string;
|
||||
};
|
||||
source: {
|
||||
l0Count: number;
|
||||
l1Count: number;
|
||||
profileCount: number;
|
||||
manifestExists: boolean;
|
||||
manifestStoreType: "sqlite" | "tcvdb" | null;
|
||||
};
|
||||
target: {
|
||||
url: string;
|
||||
username: string;
|
||||
database: string;
|
||||
alias: string;
|
||||
embeddingModel: string;
|
||||
timeout: number;
|
||||
bm25Enabled: boolean;
|
||||
bm25Language: Bm25Language;
|
||||
};
|
||||
options: {
|
||||
applyConfig: boolean;
|
||||
configBackup: boolean;
|
||||
rewriteManifest: boolean;
|
||||
failIfTargetNonempty: boolean;
|
||||
verifyCounts: boolean;
|
||||
yes: boolean;
|
||||
};
|
||||
migration?: {
|
||||
l0Migrated: number;
|
||||
l1Migrated: number;
|
||||
profileMigrated: number;
|
||||
targetL0Count: number;
|
||||
targetL1Count: number;
|
||||
targetProfileCount: number;
|
||||
configWritten: boolean;
|
||||
manifestWritten: boolean;
|
||||
manifestBackupPath?: string;
|
||||
};
|
||||
}
|
||||
|
||||
export const DEFAULT_MIGRATION_PAGE_SIZE = 50;
|
||||
|
||||
export interface MigrationTargetStore {
|
||||
init(providerInfo?: unknown): Promise<StoreInitResult> | StoreInitResult;
|
||||
isDegraded(): boolean;
|
||||
close(): void;
|
||||
upsertL1(record: MemoryRecord, embedding?: Float32Array): Promise<boolean> | boolean;
|
||||
upsertL0(record: L0Record, embedding?: Float32Array): Promise<boolean> | boolean;
|
||||
upsertL1Batch?(records: MemoryRecord[]): Promise<number>;
|
||||
upsertL0Batch?(records: L0Record[]): Promise<number>;
|
||||
countL1(): Promise<number> | number;
|
||||
countL0(): Promise<number> | number;
|
||||
pullProfiles?(): Promise<ProfileRecord[]>;
|
||||
syncProfiles?(records: ProfileSyncRecord[]): Promise<void>;
|
||||
}
|
||||
|
||||
export interface RunMigrationCliDeps {
|
||||
createTargetStore?: (options: ResolvedMigrationCliOptions) => MigrationTargetStore;
|
||||
writeMigrationPluginConfig?: typeof writeMigrationPluginConfigDefault;
|
||||
rewriteMigrationManifest?: (params: {
|
||||
dataDir: string;
|
||||
tcvdbUrl: string;
|
||||
tcvdbDatabase: string;
|
||||
tcvdbAlias?: string;
|
||||
}) => Promise<RewriteMigrationManifestResult>;
|
||||
verifyDelayMs?: number;
|
||||
}
|
||||
|
||||
function getRequiredString(values: Record<string, string | boolean | undefined>, key: string): string {
|
||||
const value = values[key];
|
||||
if (typeof value !== "string" || value.trim() === "") {
|
||||
throw new Error(`Missing required option --${key}`);
|
||||
}
|
||||
return value.trim();
|
||||
}
|
||||
|
||||
function getOptionalString(values: Record<string, string | boolean | undefined>, key: string): string | undefined {
|
||||
const value = values[key];
|
||||
if (typeof value !== "string") return undefined;
|
||||
const trimmed = value.trim();
|
||||
return trimmed === "" ? undefined : trimmed;
|
||||
}
|
||||
|
||||
function resolveBooleanOption(
|
||||
values: Record<string, string | boolean | undefined>,
|
||||
key: string,
|
||||
defaultValue: boolean,
|
||||
): boolean {
|
||||
const value = values[key];
|
||||
return typeof value === "boolean" ? value : defaultValue;
|
||||
}
|
||||
|
||||
function resolveTcvdbApiKey(values: Record<string, string | boolean | undefined>): string {
|
||||
const directApiKey = getOptionalString(values, "tcvdb-api-key");
|
||||
const apiKeyEnvName = getOptionalString(values, "tcvdb-api-key-env");
|
||||
|
||||
if (directApiKey && apiKeyEnvName) {
|
||||
throw new Error("Provide either --tcvdb-api-key or --tcvdb-api-key-env, not both");
|
||||
}
|
||||
if (directApiKey) {
|
||||
return directApiKey;
|
||||
}
|
||||
if (!apiKeyEnvName) {
|
||||
throw new Error("Missing required TCVDB API key input: use --tcvdb-api-key or --tcvdb-api-key-env");
|
||||
}
|
||||
|
||||
const envValue = process.env[apiKeyEnvName]?.trim();
|
||||
if (!envValue) {
|
||||
throw new Error(`Environment variable ${apiKeyEnvName} is empty or not set`);
|
||||
}
|
||||
return envValue;
|
||||
}
|
||||
|
||||
function parseLayers(rawLayers: string | undefined): MigrationLayer[] {
|
||||
if (!rawLayers) return [...ALL_MIGRATION_LAYERS];
|
||||
|
||||
const values = rawLayers
|
||||
.split(",")
|
||||
.map((layer) => layer.trim())
|
||||
.filter(Boolean);
|
||||
|
||||
if (values.length === 0) {
|
||||
throw new Error("--layers must include at least one layer");
|
||||
}
|
||||
|
||||
const uniqueLayers = [...new Set(values)];
|
||||
const invalid = uniqueLayers.filter(
|
||||
(layer): layer is string => !ALL_MIGRATION_LAYERS.includes(layer as MigrationLayer),
|
||||
);
|
||||
if (invalid.length > 0) {
|
||||
throw new Error(`Unsupported layer(s): ${invalid.join(", ")}`);
|
||||
}
|
||||
|
||||
return uniqueLayers as MigrationLayer[];
|
||||
}
|
||||
|
||||
function parseBm25Language(rawLanguage: string | undefined): Bm25Language {
|
||||
if (!rawLanguage) return "zh";
|
||||
if (rawLanguage === "zh" || rawLanguage === "en") {
|
||||
return rawLanguage;
|
||||
}
|
||||
throw new Error(`Unsupported --bm25-language value: ${rawLanguage}`);
|
||||
}
|
||||
|
||||
function parseTimeout(rawValue: string | undefined): number {
|
||||
if (!rawValue) return 10000;
|
||||
const timeout = Number(rawValue);
|
||||
if (!Number.isFinite(timeout) || timeout <= 0) {
|
||||
throw new Error(`Invalid --tcvdb-timeout-ms value: ${rawValue}`);
|
||||
}
|
||||
return timeout;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a "nothing to migrate" summary — used when source data dir or sqlite
|
||||
* file doesn't exist (e.g. fresh deployment that hasn't captured any data yet).
|
||||
*/
|
||||
function buildEmptySummary(options: ResolvedMigrationCliOptions): MigrationPreflightSummary {
|
||||
return {
|
||||
pluginId: options.pluginId,
|
||||
dryRun: options.dryRun,
|
||||
layers: options.layers,
|
||||
paths: {
|
||||
pluginDataDir: options.pluginDataDir,
|
||||
sqlitePath: options.sqlitePath,
|
||||
openclawConfigPath: options.openclawConfigPath,
|
||||
},
|
||||
source: {
|
||||
l0Count: 0,
|
||||
l1Count: 0,
|
||||
profileCount: 0,
|
||||
manifestExists: false,
|
||||
manifestStoreType: null,
|
||||
},
|
||||
target: {
|
||||
url: options.tcvdb.url,
|
||||
username: options.tcvdb.username,
|
||||
database: options.tcvdb.database,
|
||||
alias: options.tcvdb.alias,
|
||||
embeddingModel: options.tcvdb.embeddingModel,
|
||||
timeout: options.tcvdb.timeout,
|
||||
bm25Enabled: options.bm25Enabled,
|
||||
bm25Language: options.bm25Language,
|
||||
},
|
||||
options: {
|
||||
applyConfig: options.applyConfig,
|
||||
configBackup: options.configBackup,
|
||||
rewriteManifest: options.rewriteManifest,
|
||||
failIfTargetNonempty: options.failIfTargetNonempty,
|
||||
verifyCounts: options.verifyCounts,
|
||||
yes: options.yes,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function ensureReadablePath(filePath: string, label: string): Promise<void> {
|
||||
try {
|
||||
await fs.access(filePath);
|
||||
} catch {
|
||||
throw new Error(`${label} does not exist or is not accessible: ${filePath}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function ensureReadableDirectory(dirPath: string, label: string): Promise<void> {
|
||||
const stat = await fs.stat(dirPath).catch(() => null);
|
||||
if (!stat?.isDirectory()) {
|
||||
throw new Error(`${label} is not a directory: ${dirPath}`);
|
||||
}
|
||||
}
|
||||
|
||||
function safeParseMetadata(raw: string): Record<string, unknown> {
|
||||
if (!raw) return {};
|
||||
try {
|
||||
const parsed = JSON.parse(raw);
|
||||
return parsed && typeof parsed === "object" && !Array.isArray(parsed)
|
||||
? (parsed as Record<string, unknown>)
|
||||
: {};
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
function compactTimestamps(row: L1RecordRow): string[] {
|
||||
return [...new Set([row.timestamp_str, row.timestamp_start, row.timestamp_end].filter(Boolean))];
|
||||
}
|
||||
|
||||
function mapL1RowToMemoryRecord(row: L1RecordRow): MemoryRecord {
|
||||
const timestamps = compactTimestamps(row);
|
||||
const fallbackIso = row.updated_time || row.created_time || row.timestamp_end || row.timestamp_str || new Date(0).toISOString();
|
||||
return {
|
||||
id: row.record_id,
|
||||
content: row.content,
|
||||
type: row.type as MemoryRecord["type"],
|
||||
priority: row.priority,
|
||||
scene_name: row.scene_name,
|
||||
source_message_ids: [],
|
||||
metadata: safeParseMetadata(row.metadata_json),
|
||||
timestamps,
|
||||
createdAt: row.created_time || fallbackIso,
|
||||
updatedAt: row.updated_time || row.created_time || fallbackIso,
|
||||
sessionKey: row.session_key || "",
|
||||
sessionId: row.session_id || "",
|
||||
};
|
||||
}
|
||||
|
||||
function mapL0RowToRecord(row: L0RecordRow): L0Record {
|
||||
return {
|
||||
id: row.record_id,
|
||||
sessionKey: row.session_key,
|
||||
sessionId: row.session_id || "",
|
||||
role: row.role,
|
||||
messageText: row.message_text,
|
||||
recordedAt: row.recorded_at || "",
|
||||
timestamp: row.timestamp ?? 0,
|
||||
};
|
||||
}
|
||||
|
||||
function createTargetStoreDefault(options: ResolvedMigrationCliOptions): MigrationTargetStore {
|
||||
const bm25Encoder = createBM25Encoder(
|
||||
{
|
||||
enabled: options.bm25Enabled,
|
||||
language: options.bm25Language,
|
||||
},
|
||||
);
|
||||
|
||||
return new TcvdbMemoryStore({
|
||||
url: options.tcvdb.url,
|
||||
username: options.tcvdb.username,
|
||||
apiKey: options.tcvdb.apiKey,
|
||||
database: options.tcvdb.database,
|
||||
embeddingModel: options.tcvdb.embeddingModel,
|
||||
timeout: options.tcvdb.timeout,
|
||||
caPemPath: options.tcvdb.caPemPath,
|
||||
bm25Encoder,
|
||||
});
|
||||
}
|
||||
|
||||
async function ensureTargetIsEmpty(
|
||||
options: ResolvedMigrationCliOptions,
|
||||
targetStore: MigrationTargetStore,
|
||||
): Promise<void> {
|
||||
if (!options.failIfTargetNonempty) return;
|
||||
|
||||
const [existingL1, existingL0, existingProfiles] = await Promise.all([
|
||||
Promise.resolve(targetStore.countL1()),
|
||||
Promise.resolve(targetStore.countL0()),
|
||||
targetStore.pullProfiles ? targetStore.pullProfiles().then((records) => records.length) : Promise.resolve(0),
|
||||
]);
|
||||
|
||||
if (existingL1 > 0 || existingL0 > 0 || existingProfiles > 0) {
|
||||
throw new Error(
|
||||
`Target store is not empty (L1=${existingL1}, L0=${existingL0}, profiles=${existingProfiles})`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function migrateL1Records(sourceStore: VectorStore, targetStore: MigrationTargetStore, pageSize: number): Promise<number> {
|
||||
let migrated = 0;
|
||||
let cursor = "";
|
||||
const useBatch = typeof targetStore.upsertL1Batch === "function";
|
||||
|
||||
// eslint-disable-next-line no-constant-condition
|
||||
while (true) {
|
||||
const rows = sourceStore.queryL1RecordsCursor(cursor, pageSize);
|
||||
if (rows.length === 0) break;
|
||||
|
||||
const records = rows.map(mapL1RowToMemoryRecord);
|
||||
|
||||
if (useBatch) {
|
||||
const count = await targetStore.upsertL1Batch!(records);
|
||||
if (count === 0) {
|
||||
throw new Error(`Failed to batch migrate L1 records (cursor=${cursor}, page=${rows.length})`);
|
||||
}
|
||||
} else {
|
||||
for (const record of records) {
|
||||
const ok = await Promise.resolve(targetStore.upsertL1(record));
|
||||
if (!ok) throw new Error(`Failed to migrate L1 record ${record.id}`);
|
||||
}
|
||||
}
|
||||
|
||||
migrated += rows.length;
|
||||
cursor = rows[rows.length - 1].record_id;
|
||||
log(`L1: 已迁移 ${migrated} 条...`);
|
||||
|
||||
if (rows.length < pageSize) break;
|
||||
}
|
||||
|
||||
log(`L1: 迁移完成，共 ${migrated} 条`);
|
||||
return migrated;
|
||||
}
|
||||
|
||||
async function migrateL0Records(sourceStore: VectorStore, targetStore: MigrationTargetStore, pageSize: number): Promise<number> {
|
||||
let migrated = 0;
|
||||
let cursor = "";
|
||||
const useBatch = typeof targetStore.upsertL0Batch === "function";
|
||||
|
||||
// eslint-disable-next-line no-constant-condition
|
||||
while (true) {
|
||||
const rows = sourceStore.queryL0RecordsCursor(cursor, pageSize);
|
||||
if (rows.length === 0) break;
|
||||
|
||||
const records = rows.map(mapL0RowToRecord);
|
||||
|
||||
if (useBatch) {
|
||||
const count = await targetStore.upsertL0Batch!(records);
|
||||
if (count === 0) {
|
||||
throw new Error(`Failed to batch migrate L0 records (cursor=${cursor}, page=${rows.length})`);
|
||||
}
|
||||
} else {
|
||||
for (const record of records) {
|
||||
const ok = await Promise.resolve(targetStore.upsertL0(record));
|
||||
if (!ok) throw new Error(`Failed to migrate L0 record ${record.id}`);
|
||||
}
|
||||
}
|
||||
|
||||
migrated += rows.length;
|
||||
cursor = rows[rows.length - 1].record_id;
|
||||
log(`L0: 已迁移 ${migrated} 条...`);
|
||||
|
||||
if (rows.length < pageSize) break;
|
||||
}
|
||||
|
||||
log(`L0: 迁移完成，共 ${migrated} 条`);
|
||||
return migrated;
|
||||
}
|
||||
|
||||
async function migrateProfiles(
|
||||
pluginDataDir: string,
|
||||
targetStore: MigrationTargetStore,
|
||||
): Promise<number> {
|
||||
const profiles = await listLocalProfiles(pluginDataDir);
|
||||
if (profiles.length === 0) {
|
||||
log("Profiles: 无本地 profile，跳过");
|
||||
return 0;
|
||||
}
|
||||
if (!targetStore.syncProfiles) {
|
||||
throw new Error("Target store does not support profile sync");
|
||||
}
|
||||
log(`Profiles: 发现 ${profiles.length} 个本地 profile，开始同步...`);
|
||||
await targetStore.syncProfiles(profiles);
|
||||
log(`Profiles: 同步完成，共 ${profiles.length} 个`);
|
||||
return profiles.length;
|
||||
}
|
||||
|
||||
async function verifyMigratedCounts(
|
||||
summary: MigrationPreflightSummary,
|
||||
targetStore: MigrationTargetStore,
|
||||
delayMs: number,
|
||||
): Promise<{ l1Count: number; l0Count: number; profileCount: number }> {
|
||||
if (delayMs > 0) {
|
||||
log(`等待 ${Math.round(delayMs / 1000)} 秒让远端数据落盘...`);
|
||||
await sleep(delayMs);
|
||||
}
|
||||
log("开始校验迁移数量...");
|
||||
|
||||
const [l1Count, l0Count, profileCount] = await Promise.all([
|
||||
Promise.resolve(targetStore.countL1()),
|
||||
Promise.resolve(targetStore.countL0()),
|
||||
targetStore.pullProfiles ? targetStore.pullProfiles().then((records) => records.length) : Promise.resolve(0),
|
||||
]);
|
||||
|
||||
log(`校验结果: L1=${l1Count}/${summary.source.l1Count}, L0=${l0Count}/${summary.source.l0Count}, Profiles=${profileCount}/${summary.source.profileCount}`);
|
||||
|
||||
if (l1Count !== summary.source.l1Count) {
|
||||
throw new Error(`L1 count verification failed: source=${summary.source.l1Count}, target=${l1Count}`);
|
||||
}
|
||||
if (l0Count !== summary.source.l0Count) {
|
||||
throw new Error(`L0 count verification failed: source=${summary.source.l0Count}, target=${l0Count}`);
|
||||
}
|
||||
if (profileCount !== summary.source.profileCount) {
|
||||
throw new Error(
|
||||
`Profile count verification failed: source=${summary.source.profileCount}, target=${profileCount}`,
|
||||
);
|
||||
}
|
||||
|
||||
log("校验通过");
|
||||
return { l1Count, l0Count, profileCount };
|
||||
}
|
||||
|
||||
async function writeSummaryJson(
|
||||
summaryJsonPath: string | undefined,
|
||||
summary: MigrationPreflightSummary,
|
||||
): Promise<void> {
|
||||
if (!summaryJsonPath) return;
|
||||
await fs.mkdir(path.dirname(summaryJsonPath), { recursive: true });
|
||||
await fs.writeFile(summaryJsonPath, `${JSON.stringify(summary, null, 2)}\n`, "utf-8");
|
||||
}
|
||||
|
||||
function printUsageAndExit(): never {
|
||||
const text = `
|
||||
SQLite → 腾讯云向量数据库迁移工具
|
||||
|
||||
Usage:
|
||||
migrate-sqlite-to-tcvdb [options]
|
||||
|
||||
Required:
|
||||
--plugin-data-dir <path> 插件数据目录路径
|
||||
--openclaw-config-path <path> openclaw.json 配置文件路径
|
||||
--tcvdb-url <url> TCVDB 服务地址
|
||||
--tcvdb-username <name> TCVDB 用户名
|
||||
--tcvdb-database <name> TCVDB 数据库名
|
||||
--tcvdb-embedding-model <name> Embedding 模型名称
|
||||
--tcvdb-api-key <key> TCVDB API 密钥（明文，与 --tcvdb-api-key-env 二选一）
|
||||
--tcvdb-api-key-env <var> 包含 API 密钥的环境变量名
|
||||
|
||||
Optional:
|
||||
--sqlite-path <path> SQLite 数据库路径（默认: <plugin-data-dir>/vectors.db）
|
||||
--plugin-id <id> 写入配置时使用的插件 ID（默认: memory-tencentdb）
|
||||
--layers <l0,l1,l2,l3> 要迁移的层，逗号分隔（默认: l0,l1,l2,l3）
|
||||
--tcvdb-alias <name> 用户自定义别名
|
||||
--tcvdb-timeout-ms <ms> 请求超时（默认: 10000）
|
||||
--tcvdb-ca-pem <path> CA 证书 PEM 文件路径（HTTPS 连接时使用）
|
||||
--bm25-language <zh|en> BM25 分词语言（默认: zh）
|
||||
--summary-json-path <path> 将迁移摘要写入此文件
|
||||
--job-id <id> 迁移任务 ID（用于追踪）
|
||||
|
||||
Flags:
|
||||
--dry-run 仅预览，不执行写入
|
||||
--yes 跳过交互确认
|
||||
--no-apply-config 不自动更新 openclaw.json
|
||||
--no-config-backup 写入配置前不备份
|
||||
--no-rewrite-manifest 不更新 manifest.json
|
||||
--no-fail-if-target-nonempty 目标库非空时不中止
|
||||
--no-verify-counts 迁移后不校验记录数
|
||||
--no-bm25-enabled 禁用 BM25 稀疏向量
|
||||
|
||||
-h, --help 显示此帮助信息
|
||||
|
||||
Examples:
|
||||
# 预检模式
|
||||
migrate-sqlite-to-tcvdb \\
|
||||
--plugin-data-dir ~/.openclaw/memory-tdai \\
|
||||
--openclaw-config-path ~/.openclaw/openclaw.json \\
|
||||
--tcvdb-url http://127.0.0.1:80 --tcvdb-username root \\
|
||||
--tcvdb-api-key-env TCVDB_API_KEY \\
|
||||
--tcvdb-database agent_memory_prod \\
|
||||
--tcvdb-embedding-model bge-large-zh \\
|
||||
--dry-run
|
||||
|
||||
# 正式迁移
|
||||
migrate-sqlite-to-tcvdb \\
|
||||
--plugin-data-dir ~/.openclaw/memory-tdai \\
|
||||
--openclaw-config-path ~/.openclaw/openclaw.json \\
|
||||
--tcvdb-url http://127.0.0.1:80 --tcvdb-username root \\
|
||||
--tcvdb-api-key-env TCVDB_API_KEY \\
|
||||
--tcvdb-database agent_memory_prod \\
|
||||
--tcvdb-embedding-model bge-large-zh \\
|
||||
--yes
|
||||
`.trimStart();
|
||||
process.stdout.write(text);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
export function resolveMigrationCliOptions(argv: string[]): ResolvedMigrationCliOptions {
|
||||
const { values } = parseArgs({
|
||||
args: argv,
|
||||
strict: true,
|
||||
allowPositionals: false,
|
||||
allowNegative: true,
|
||||
options: {
|
||||
help: { type: "boolean", short: "h" },
|
||||
"plugin-data-dir": { type: "string" },
|
||||
"sqlite-path": { type: "string" },
|
||||
"openclaw-config-path": { type: "string" },
|
||||
"plugin-id": { type: "string" },
|
||||
layers: { type: "string" },
|
||||
"apply-config": { type: "boolean" },
|
||||
"config-backup": { type: "boolean" },
|
||||
"rewrite-manifest": { type: "boolean" },
|
||||
"fail-if-target-nonempty": { type: "boolean" },
|
||||
"verify-counts": { type: "boolean" },
|
||||
"dry-run": { type: "boolean" },
|
||||
yes: { type: "boolean" },
|
||||
"bm25-enabled": { type: "boolean" },
|
||||
"bm25-language": { type: "string" },
|
||||
"summary-json-path": { type: "string" },
|
||||
"job-id": { type: "string" },
|
||||
"tcvdb-url": { type: "string" },
|
||||
"tcvdb-username": { type: "string" },
|
||||
"tcvdb-api-key": { type: "string" },
|
||||
"tcvdb-api-key-env": { type: "string" },
|
||||
"tcvdb-database": { type: "string" },
|
||||
"tcvdb-alias": { type: "string" },
|
||||
"tcvdb-embedding-model": { type: "string" },
|
||||
"tcvdb-timeout-ms": { type: "string" },
|
||||
"tcvdb-ca-pem": { type: "string" },
|
||||
},
|
||||
});
|
||||
|
||||
if (values.help) {
|
||||
printUsageAndExit();
|
||||
}
|
||||
|
||||
const pluginDataDir = path.resolve(getRequiredString(values, "plugin-data-dir"));
|
||||
const sqlitePath = path.resolve(
|
||||
getOptionalString(values, "sqlite-path") ?? path.join(pluginDataDir, "vectors.db"),
|
||||
);
|
||||
const summaryJsonPath = getOptionalString(values, "summary-json-path");
|
||||
|
||||
return {
|
||||
pluginDataDir,
|
||||
sqlitePath,
|
||||
openclawConfigPath: path.resolve(getRequiredString(values, "openclaw-config-path")),
|
||||
pluginId: getOptionalString(values, "plugin-id") ?? DEFAULT_MIGRATION_PLUGIN_ID,
|
||||
layers: parseLayers(getOptionalString(values, "layers")),
|
||||
applyConfig: resolveBooleanOption(values, "apply-config", true),
|
||||
configBackup: resolveBooleanOption(values, "config-backup", true),
|
||||
rewriteManifest: resolveBooleanOption(values, "rewrite-manifest", true),
|
||||
failIfTargetNonempty: resolveBooleanOption(values, "fail-if-target-nonempty", true),
|
||||
verifyCounts: resolveBooleanOption(values, "verify-counts", true),
|
||||
dryRun: resolveBooleanOption(values, "dry-run", false),
|
||||
yes: resolveBooleanOption(values, "yes", false),
|
||||
bm25Enabled: resolveBooleanOption(values, "bm25-enabled", true),
|
||||
bm25Language: parseBm25Language(getOptionalString(values, "bm25-language")),
|
||||
summaryJsonPath: summaryJsonPath ? path.resolve(summaryJsonPath) : undefined,
|
||||
jobId: getOptionalString(values, "job-id"),
|
||||
tcvdb: {
|
||||
url: getRequiredString(values, "tcvdb-url"),
|
||||
username: getRequiredString(values, "tcvdb-username"),
|
||||
apiKey: resolveTcvdbApiKey(values),
|
||||
database: getRequiredString(values, "tcvdb-database"),
|
||||
alias: getOptionalString(values, "tcvdb-alias") ?? "",
|
||||
embeddingModel: getRequiredString(values, "tcvdb-embedding-model"),
|
||||
timeout: parseTimeout(getOptionalString(values, "tcvdb-timeout-ms")),
|
||||
caPemPath: getOptionalString(values, "tcvdb-ca-pem"),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export async function collectMigrationPreflight(
|
||||
options: ResolvedMigrationCliOptions,
|
||||
): Promise<MigrationPreflightSummary> {
|
||||
// ── 优雅处理"数据目录 / sqlite 不存在"的场景 ──
|
||||
// 如果源数据路径不存在（全新部署、尚未有任何 capture），不报错——
|
||||
// 返回"全零"summary，让调用方判断是否跳过迁移。
|
||||
const dirExists = await fs.stat(options.pluginDataDir).then(s => s.isDirectory()).catch(() => false);
|
||||
if (!dirExists) {
|
||||
log(`plugin data directory 不存在，无需迁移: ${options.pluginDataDir}`);
|
||||
return buildEmptySummary(options);
|
||||
}
|
||||
const sqliteExists = await fs.access(options.sqlitePath).then(() => true).catch(() => false);
|
||||
if (!sqliteExists) {
|
||||
log(`sqlite database 不存在，无需迁移: ${options.sqlitePath}`);
|
||||
return buildEmptySummary(options);
|
||||
}
|
||||
await ensureReadablePath(options.openclawConfigPath, "OpenClaw config file");
|
||||
|
||||
const store = new VectorStore(options.sqlitePath, 0);
|
||||
const initResult = store.init();
|
||||
if (store.isDegraded()) {
|
||||
store.close();
|
||||
throw new Error(`Failed to open sqlite store for migration preflight: ${initResult.reason ?? "unknown error"}`);
|
||||
}
|
||||
|
||||
try {
|
||||
const [profiles, manifest] = await Promise.all([
|
||||
listLocalProfiles(options.pluginDataDir),
|
||||
Promise.resolve(readManifest(options.pluginDataDir)),
|
||||
]);
|
||||
|
||||
return {
|
||||
pluginId: options.pluginId,
|
||||
dryRun: options.dryRun,
|
||||
layers: options.layers,
|
||||
paths: {
|
||||
pluginDataDir: options.pluginDataDir,
|
||||
sqlitePath: options.sqlitePath,
|
||||
openclawConfigPath: options.openclawConfigPath,
|
||||
},
|
||||
source: {
|
||||
l0Count: store.countL0(),
|
||||
l1Count: store.countL1(),
|
||||
profileCount: profiles.length,
|
||||
manifestExists: manifest !== null,
|
||||
manifestStoreType: manifest?.store.type ?? null,
|
||||
},
|
||||
target: {
|
||||
url: options.tcvdb.url,
|
||||
username: options.tcvdb.username,
|
||||
database: options.tcvdb.database,
|
||||
alias: options.tcvdb.alias,
|
||||
embeddingModel: options.tcvdb.embeddingModel,
|
||||
timeout: options.tcvdb.timeout,
|
||||
bm25Enabled: options.bm25Enabled,
|
||||
bm25Language: options.bm25Language,
|
||||
},
|
||||
options: {
|
||||
applyConfig: options.applyConfig,
|
||||
configBackup: options.configBackup,
|
||||
rewriteManifest: options.rewriteManifest,
|
||||
failIfTargetNonempty: options.failIfTargetNonempty,
|
||||
verifyCounts: options.verifyCounts,
|
||||
yes: options.yes,
|
||||
},
|
||||
};
|
||||
} finally {
|
||||
store.close();
|
||||
}
|
||||
}
|
||||
|
||||
export async function runMigrationCli(
|
||||
argv: string[],
|
||||
deps: RunMigrationCliDeps = {},
|
||||
): Promise<MigrationPreflightSummary> {
|
||||
const options = resolveMigrationCliOptions(argv);
|
||||
log("开始预检...");
|
||||
const summary = await collectMigrationPreflight(options);
|
||||
log(`预检完成: 源数据 L1=${summary.source.l1Count}, L0=${summary.source.l0Count}, Profiles=${summary.source.profileCount}`);
|
||||
log(`目标: ${summary.target.url} / ${summary.target.database}`);
|
||||
|
||||
const hasSourceData = summary.source.l0Count > 0 || summary.source.l1Count > 0 || summary.source.profileCount > 0;
|
||||
|
||||
if (!hasSourceData) {
|
||||
log("源数据为空，跳过数据迁移。");
|
||||
}
|
||||
|
||||
if (options.dryRun) {
|
||||
log("预检模式 (dry-run)，不执行写入");
|
||||
await writeSummaryJson(options.summaryJsonPath, summary);
|
||||
return summary;
|
||||
}
|
||||
|
||||
const createTargetStore = deps.createTargetStore ?? createTargetStoreDefault;
|
||||
const writeMigrationPluginConfig = deps.writeMigrationPluginConfig ?? writeMigrationPluginConfigDefault;
|
||||
const rewriteMigrationManifest = deps.rewriteMigrationManifest ?? (async (params) =>
|
||||
await rewriteMigrationManifestDefault(params));
|
||||
|
||||
const migration = {
|
||||
l1Migrated: 0,
|
||||
l0Migrated: 0,
|
||||
profileMigrated: 0,
|
||||
targetL1Count: 0,
|
||||
targetL0Count: 0,
|
||||
targetProfileCount: 0,
|
||||
configWritten: false,
|
||||
manifestWritten: false,
|
||||
manifestBackupPath: undefined as string | undefined,
|
||||
};
|
||||
|
||||
if (hasSourceData) {
|
||||
log("打开 SQLite 源库...");
|
||||
const sourceStore = new VectorStore(options.sqlitePath, 0);
|
||||
const sourceInitResult = sourceStore.init();
|
||||
if (sourceStore.isDegraded()) {
|
||||
sourceStore.close();
|
||||
throw new Error(`Failed to reopen sqlite source store: ${sourceInitResult.reason ?? "unknown error"}`);
|
||||
}
|
||||
|
||||
log("初始化目标库...");
|
||||
const targetStore = createTargetStore(options);
|
||||
|
||||
try {
|
||||
await Promise.resolve(targetStore.init());
|
||||
if (targetStore.isDegraded()) {
|
||||
throw new Error("Target store entered degraded mode during initialization");
|
||||
}
|
||||
|
||||
await ensureTargetIsEmpty(options, targetStore);
|
||||
|
||||
const pageSize = DEFAULT_MIGRATION_PAGE_SIZE;
|
||||
log(`分页大小: ${pageSize} 条/批`);
|
||||
|
||||
migration.l1Migrated = options.layers.includes("l1") ? await migrateL1Records(sourceStore, targetStore, pageSize) : 0;
|
||||
migration.l0Migrated = options.layers.includes("l0") ? await migrateL0Records(sourceStore, targetStore, pageSize) : 0;
|
||||
migration.profileMigrated = options.layers.includes("l2") || options.layers.includes("l3")
|
||||
? await migrateProfiles(options.pluginDataDir, targetStore)
|
||||
: 0;
|
||||
|
||||
const verifyDelayMs = deps.verifyDelayMs ?? 10_000;
|
||||
|
||||
const verifiedCounts = options.verifyCounts
|
||||
? await verifyMigratedCounts(summary, targetStore, verifyDelayMs)
|
||||
: {
|
||||
l1Count: await Promise.resolve(targetStore.countL1()),
|
||||
l0Count: await Promise.resolve(targetStore.countL0()),
|
||||
profileCount: targetStore.pullProfiles ? (await targetStore.pullProfiles()).length : 0,
|
||||
};
|
||||
|
||||
migration.targetL1Count = verifiedCounts.l1Count;
|
||||
migration.targetL0Count = verifiedCounts.l0Count;
|
||||
migration.targetProfileCount = verifiedCounts.profileCount;
|
||||
} finally {
|
||||
sourceStore.close();
|
||||
targetStore.close();
|
||||
}
|
||||
}
|
||||
|
||||
if (options.applyConfig) {
|
||||
log(`写入配置到 ${options.openclawConfigPath} ...`);
|
||||
await writeMigrationPluginConfig({
|
||||
configPath: options.openclawConfigPath,
|
||||
pluginId: options.pluginId,
|
||||
tcvdb: {
|
||||
url: options.tcvdb.url,
|
||||
username: options.tcvdb.username,
|
||||
apiKey: options.tcvdb.apiKey,
|
||||
database: options.tcvdb.database,
|
||||
alias: options.tcvdb.alias,
|
||||
embeddingModel: options.tcvdb.embeddingModel,
|
||||
timeout: options.tcvdb.timeout,
|
||||
},
|
||||
bm25: {
|
||||
enabled: options.bm25Enabled,
|
||||
language: options.bm25Language,
|
||||
},
|
||||
});
|
||||
migration.configWritten = true;
|
||||
log("配置写入完成");
|
||||
}
|
||||
|
||||
if (options.rewriteManifest) {
|
||||
log("更新 manifest...");
|
||||
const manifestResult = await rewriteMigrationManifest({
|
||||
dataDir: options.pluginDataDir,
|
||||
tcvdbUrl: options.tcvdb.url,
|
||||
tcvdbDatabase: options.tcvdb.database,
|
||||
tcvdbAlias: options.tcvdb.alias || undefined,
|
||||
});
|
||||
migration.manifestWritten = manifestResult.created || manifestResult.updated;
|
||||
migration.manifestBackupPath = manifestResult.backupPath;
|
||||
log(`Manifest ${manifestResult.created ? "创建" : "更新"}完成${manifestResult.backupPath ? `，备份: ${manifestResult.backupPath}` : ""}`);
|
||||
}
|
||||
|
||||
summary.migration = {
|
||||
l1Migrated: migration.l1Migrated,
|
||||
l0Migrated: migration.l0Migrated,
|
||||
profileMigrated: migration.profileMigrated,
|
||||
targetL1Count: migration.targetL1Count,
|
||||
targetL0Count: migration.targetL0Count,
|
||||
targetProfileCount: migration.targetProfileCount,
|
||||
configWritten: migration.configWritten,
|
||||
manifestWritten: migration.manifestWritten,
|
||||
manifestBackupPath: migration.manifestBackupPath,
|
||||
};
|
||||
|
||||
await writeSummaryJson(options.summaryJsonPath, summary);
|
||||
log("迁移全部完成!");
|
||||
return summary;
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "es2023",
|
||||
"module": "NodeNext",
|
||||
"moduleResolution": "NodeNext",
|
||||
"declaration": false,
|
||||
"strict": true,
|
||||
"esModuleInterop": true,
|
||||
"skipLibCheck": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"resolveJsonModule": true,
|
||||
"types": ["node"],
|
||||
"paths": {
|
||||
"node-llama-cpp": ["./node-llama-cpp.d.ts"]
|
||||
},
|
||||
"outDir": "dist",
|
||||
"rootDir": "../.."
|
||||
},
|
||||
"files": [
|
||||
"cli-entry.ts",
|
||||
"sqlite-to-tcvdb.ts",
|
||||
"config-write.ts",
|
||||
"manifest-write.ts"
|
||||
]
|
||||
}
|
||||
Reference in New Issue
Block a user