mirror of
https://github.com/TencentCloud/TencentDB-Agent-Memory
synced 2026-07-10 20:34:30 +00:00
feat: release v0.3.4 — offload local LLM, CLI restore, bugfix scripts
This commit is contained in:
chrishuan
@@ -0,0 +1,174 @@
|
||||
# memory-tdai CLI
|
||||
|
||||
`openclaw memory-tdai` 命令空间，提供离线数据管理工具。
|
||||
|
||||
## seed — 导入历史对话数据
|
||||
|
||||
将历史对话 JSON 文件导入到记忆管线中，完整执行 L0→L1→L2→L3 流程。适用于：
|
||||
|
||||
- 将已有对话数据灌入记忆系统
|
||||
- 批量测试记忆提取效果
|
||||
- 迁移/恢复记忆数据
|
||||
|
||||
### 用法
|
||||
|
||||
```bash
|
||||
openclaw memory-tdai seed --input <file> [options]
|
||||
```
|
||||
|
||||
### 参数
|
||||
|
||||
| 参数 | 必填 | 说明 |
|
||||
|------|------|------|
|
||||
| `--input <file>` | ✅ | 输入 JSON 文件路径 |
|
||||
| `--output-dir <dir>` | — | 输出目录（默认自动生成带时间戳的目录） |
|
||||
| `--session-key <key>` | — | 回退 session key（当输入数据缺少时使用） |
|
||||
| `--config <file>` | — | 配置覆盖文件（JSON，与 openclaw.json 插件配置深度合并） |
|
||||
| `--strict-round-role` | — | 严格校验每轮对话必须包含 user 和 assistant 消息 |
|
||||
| `--yes` | — | 跳过交互确认（如时间戳自动填充确认） |
|
||||
|
||||
### 示例
|
||||
|
||||
```bash
|
||||
# 基本用法
|
||||
openclaw memory-tdai seed --input conversations.json
|
||||
|
||||
# 指定输出目录
|
||||
openclaw memory-tdai seed --input data.json --output-dir ./seed-output
|
||||
|
||||
# 使用自定义配置覆盖（如调整 pipeline 参数）
|
||||
openclaw memory-tdai seed --input data.json --config seed-config.json
|
||||
|
||||
# 跳过所有确认
|
||||
openclaw memory-tdai seed --input data.json --yes
|
||||
|
||||
# 严格模式 + 自定义配置
|
||||
openclaw memory-tdai seed --input data.json --config seed-config.json --strict-round-role --yes
|
||||
```
|
||||
|
||||
### 输入文件格式
|
||||
|
||||
支持两种 JSON 格式：
|
||||
|
||||
#### Format A：对象包装
|
||||
|
||||
```json
|
||||
{
|
||||
"sessions": [
|
||||
{
|
||||
"sessionKey": "user-alice",
|
||||
"sessionId": "conv-001",
|
||||
"conversations": [
|
||||
[
|
||||
{ "role": "user", "content": "你好", "timestamp": 1711929600000 },
|
||||
{ "role": "assistant", "content": "你好！有什么可以帮你？", "timestamp": 1711929601000 }
|
||||
],
|
||||
[
|
||||
{ "role": "user", "content": "今天天气怎么样？" },
|
||||
{ "role": "assistant", "content": "今天晴天，适合出门。" }
|
||||
]
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
#### Format B：顶层数组
|
||||
|
||||
```json
|
||||
[
|
||||
{
|
||||
"sessionKey": "user-alice",
|
||||
"conversations": [
|
||||
[
|
||||
{ "role": "user", "content": "你好" },
|
||||
{ "role": "assistant", "content": "你好！" }
|
||||
]
|
||||
]
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
#### 字段说明
|
||||
|
||||
| 字段 | 类型 | 必填 | 说明 |
|
||||
|------|------|------|------|
|
||||
| `sessionKey` | string | ✅ | Session 标识（如用户 ID、频道名） |
|
||||
| `sessionId` | string | — | 会话实例 ID（同一 sessionKey 下可有多个 sessionId） |
|
||||
| `conversations` | message[][] | ✅ | 对话轮次数组，每个轮次是一组消息 |
|
||||
| `role` | string | ✅ | 消息角色：`user` 或 `assistant` |
|
||||
| `content` | string | ✅ | 消息内容 |
|
||||
| `timestamp` | number \| string | — | 时间戳：epoch 毫秒或 ISO 8601 字符串。缺失时 seed 会提示自动填充 |
|
||||
|
||||
### 配置覆盖
|
||||
|
||||
`--config` 接受一个 JSON 文件，与 `openclaw.json` 中的插件配置**两级深度合并**：
|
||||
|
||||
- 顶层 key 如果两边都是对象 → 浅合并（保留 base 中未覆盖的字段）
|
||||
- 其他类型 → 直接覆盖
|
||||
|
||||
常见场景：seed 时使用更激进的 pipeline 参数以加速处理：
|
||||
|
||||
```json
|
||||
{
|
||||
"pipeline": {
|
||||
"everyNConversations": 3,
|
||||
"enableWarmup": false,
|
||||
"l1IdleTimeoutSeconds": 2,
|
||||
"l2DelayAfterL1Seconds": 1,
|
||||
"l2MinIntervalSeconds": 1,
|
||||
"l2MaxIntervalSeconds": 10
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
如果需要 seed 到独立的 TCVDB 数据库：
|
||||
|
||||
```json
|
||||
{
|
||||
"storeBackend": "tcvdb",
|
||||
"tcvdb": {
|
||||
"database": "my_seed_test_db"
|
||||
},
|
||||
"pipeline": {
|
||||
"everyNConversations": 3,
|
||||
"enableWarmup": false,
|
||||
"l1IdleTimeoutSeconds": 2
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 输出目录结构
|
||||
|
||||
```
|
||||
<output-dir>/
|
||||
├── conversations/ — L0 JSONL 文件
|
||||
├── records/ — L1 JSONL 文件
|
||||
├── scene_blocks/ — L2 场景块
|
||||
├── vectors.db — SQLite 向量数据库（仅 sqlite 后端）
|
||||
├── .metadata/
|
||||
│ ├── manifest.json — 元数据（store 绑定 + seed 运行记录）
|
||||
│ └── checkpoint.json — 管线进度
|
||||
└── .backup/ — 滚动备份
|
||||
```
|
||||
|
||||
Seed 完成后，`manifest.json` 会记录本次运行信息：
|
||||
|
||||
```json
|
||||
{
|
||||
"version": 1,
|
||||
"createdAt": "2026-04-01T22:00:00.000Z",
|
||||
"store": {
|
||||
"type": "sqlite",
|
||||
"sqlite": { "path": "vectors.db" }
|
||||
},
|
||||
"seed": {
|
||||
"inputFile": "conversations.json",
|
||||
"sessions": 3,
|
||||
"rounds": 42,
|
||||
"messages": 128,
|
||||
"startedAt": "2026-04-01T22:00:00.000Z",
|
||||
"completedAt": "2026-04-01T22:05:30.000Z"
|
||||
}
|
||||
}
|
||||
```
|
||||
@@ -0,0 +1,281 @@
|
||||
/**
|
||||
* `openclaw memory-tdai seed` command definition.
|
||||
*
|
||||
* Responsibilities:
|
||||
* - Define CLI parameters and help text
|
||||
* - Interactive confirmation for timestamp auto-fill
|
||||
* - Output directory resolution and checkpoint detection
|
||||
* - Delegate to seed-runtime for actual execution
|
||||
*/
|
||||
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import readline from "node:readline";
|
||||
import type { Command } from "commander";
|
||||
import type { SeedCliContext } from "../index.ts";
|
||||
import type { SeedCommandOptions } from "../../core/seed/types.js";
|
||||
import { loadAndValidateInput, fillTimestamps, SeedValidationError } from "../../core/seed/input.js";
|
||||
import { executeSeed } from "../../core/seed/seed-runtime.js";
|
||||
|
||||
const TAG = "[memory-tdai] [seed-cmd]";
|
||||
|
||||
/**
|
||||
* Register the `seed` subcommand under the memory-tdai CLI namespace.
|
||||
*/
|
||||
export function registerSeedCommand(parent: Command, ctx: SeedCliContext): void {
|
||||
parent
|
||||
.command("seed")
|
||||
.description("Seed historical conversation data into the memory pipeline (L0 → L1)")
|
||||
.requiredOption("--input <file>", "Path to input JSON file")
|
||||
.option("--output-dir <dir>", "Output directory for pipeline data (default: auto-generated)")
|
||||
.option("--session-key <key>", "Fallback session key when input lacks one")
|
||||
.option("--config <file>", "Path to memory-tdai config override file (JSON, deep-merged on top of current plugin config)")
|
||||
.option("--strict-round-role", "Require each round to have both user and assistant messages", false)
|
||||
.option("--yes", "Skip interactive confirmations (e.g. timestamp auto-fill)", false)
|
||||
.addHelpText("after", `
|
||||
Examples:
|
||||
openclaw memory-tdai seed --input conversations.json
|
||||
openclaw memory-tdai seed --input data.json --output-dir ./seed-output --strict-round-role
|
||||
openclaw memory-tdai seed --input data.json --config ./seed-config.json
|
||||
openclaw memory-tdai seed --input data.json --yes
|
||||
`)
|
||||
.action(async (rawOpts: Record<string, unknown>) => {
|
||||
const opts: SeedCommandOptions = {
|
||||
input: rawOpts.input as string,
|
||||
outputDir: rawOpts.outputDir as string | undefined,
|
||||
sessionKey: rawOpts.sessionKey as string | undefined,
|
||||
strictRoundRole: rawOpts.strictRoundRole === true,
|
||||
yes: rawOpts.yes === true,
|
||||
configFile: rawOpts.config as string | undefined,
|
||||
};
|
||||
|
||||
await runSeedCommand(opts, ctx);
|
||||
});
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Command handler
|
||||
// ============================
|
||||
|
||||
async function runSeedCommand(opts: SeedCommandOptions, ctx: SeedCliContext): Promise<void> {
|
||||
const { logger } = ctx;
|
||||
|
||||
logger.info(`${TAG} Starting seed command...`);
|
||||
logger.info(`${TAG} input: ${opts.input}`);
|
||||
logger.info(`${TAG} outputDir: ${opts.outputDir ?? "(auto)"}`);
|
||||
logger.info(`${TAG} sessionKey: ${opts.sessionKey ?? "(from input)"}`);
|
||||
logger.info(`${TAG} config: ${opts.configFile ?? "(default)"}`);
|
||||
logger.info(`${TAG} strict: ${opts.strictRoundRole}`);
|
||||
logger.info(`${TAG} yes: ${opts.yes}`);
|
||||
|
||||
// 0. Load config override file and deep-merge with base plugin config
|
||||
const mergedPluginConfig = loadAndMergePluginConfig(
|
||||
ctx.pluginConfig as Record<string, unknown> | undefined,
|
||||
opts.configFile,
|
||||
logger,
|
||||
);
|
||||
|
||||
// 1. Load and validate input
|
||||
let loadResult;
|
||||
try {
|
||||
loadResult = loadAndValidateInput(opts);
|
||||
} catch (err) {
|
||||
if (err instanceof SeedValidationError) {
|
||||
console.error(`\n❌ ${err.message}\n`);
|
||||
process.exit(1);
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
|
||||
const { input, needsTimestampConfirmation } = loadResult;
|
||||
|
||||
console.log(
|
||||
`\n📥 Input loaded: ${input.sessions.length} session(s), ` +
|
||||
`${input.totalRounds} round(s), ${input.totalMessages} message(s)` +
|
||||
`${input.hasTimestamps ? "" : " (no timestamps)"}`,
|
||||
);
|
||||
|
||||
// 2. Timestamp confirmation (if all messages lack timestamps)
|
||||
if (needsTimestampConfirmation) {
|
||||
if (opts.yes) {
|
||||
console.log(" Timestamps missing — auto-filling with current time (--yes)");
|
||||
fillTimestamps(input);
|
||||
} else {
|
||||
const confirmed = await askConfirmation(
|
||||
"All messages have no timestamp. Use current time for each conversation round? [y/N] ",
|
||||
);
|
||||
if (!confirmed) {
|
||||
console.log("Aborted.");
|
||||
process.exit(0);
|
||||
}
|
||||
fillTimestamps(input);
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Resolve output directory
|
||||
const outputDir = resolveOutputDir(opts.outputDir, ctx.stateDir);
|
||||
logger.info(`${TAG} Output directory: ${outputDir}`);
|
||||
|
||||
// 4. Check for existing directory / checkpoint (resume detection)
|
||||
if (fs.existsSync(outputDir)) {
|
||||
const checkpointPath = path.join(outputDir, ".metadata", "checkpoint.json");
|
||||
if (fs.existsSync(checkpointPath)) {
|
||||
// Checkpoint exists → resume scenario → P0 not implemented
|
||||
console.error(
|
||||
"\n❌ Resume from checkpoint is not implemented in P0 yet. " +
|
||||
"Please use a new output directory.\n" +
|
||||
` Existing: ${outputDir}\n`,
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Directory exists but no checkpoint → might have stale data
|
||||
const entries = fs.readdirSync(outputDir);
|
||||
if (entries.length > 0) {
|
||||
console.error(
|
||||
`\n❌ Output directory already exists and is not empty: ${outputDir}\n` +
|
||||
" Please use a new directory or clean the existing one.\n",
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
// 5. Execute seed pipeline
|
||||
console.log(`\n🔧 Output: ${outputDir}`);
|
||||
console.log(`▶️ Starting seed pipeline...\n`);
|
||||
|
||||
const summary = await executeSeed(input, {
|
||||
outputDir,
|
||||
openclawConfig: ctx.config,
|
||||
pluginConfig: mergedPluginConfig,
|
||||
inputFile: opts.input,
|
||||
logger,
|
||||
onProgress: (progress) => {
|
||||
const pct = ((progress.currentRound / progress.totalRounds) * 100).toFixed(0);
|
||||
process.stdout.write(
|
||||
`\r [${progress.currentRound}/${progress.totalRounds}] ${pct}% ` +
|
||||
`session=${progress.sessionKey} stage=${progress.stage} `,
|
||||
);
|
||||
},
|
||||
});
|
||||
|
||||
// 6. Print summary
|
||||
console.log("\n");
|
||||
console.log("╔══════════════════════════════════════════╗");
|
||||
console.log("║ Seed Summary ║");
|
||||
console.log("╠══════════════════════════════════════════╣");
|
||||
console.log(`║ Sessions: ${String(summary.sessionsProcessed).padStart(11)} ║`);
|
||||
console.log(`║ Rounds: ${String(summary.roundsProcessed).padStart(11)} ║`);
|
||||
console.log(`║ Messages: ${String(summary.messagesProcessed).padStart(11)} ║`);
|
||||
console.log(`║ L0 recorded: ${String(summary.l0RecordedCount).padStart(11)} ║`);
|
||||
console.log(`║ Duration: ${(summary.durationMs / 1000).toFixed(1).padStart(10)}s ║`);
|
||||
console.log("╚══════════════════════════════════════════╝");
|
||||
console.log(`\n📁 Output: ${summary.outputDir}\n`);
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Helpers
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Load an optional config override file and deep-merge it on top of the
|
||||
* base plugin config from openclaw.json.
|
||||
*
|
||||
* Returns the merged config, or the base config unchanged if no override
|
||||
* file is specified.
|
||||
*/
|
||||
function loadAndMergePluginConfig(
|
||||
base: Record<string, unknown> | undefined,
|
||||
configFile: string | undefined,
|
||||
logger: { info: (msg: string) => void },
|
||||
): Record<string, unknown> | undefined {
|
||||
if (!configFile) return base;
|
||||
|
||||
const resolved = path.resolve(configFile);
|
||||
if (!fs.existsSync(resolved)) {
|
||||
console.error(`\n❌ Config override file not found: ${resolved}\n`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
let override: Record<string, unknown>;
|
||||
try {
|
||||
const raw = fs.readFileSync(resolved, "utf-8");
|
||||
override = JSON.parse(raw) as Record<string, unknown>;
|
||||
} catch (err) {
|
||||
console.error(
|
||||
`\n❌ Failed to parse config override file: ${resolved}\n` +
|
||||
` ${err instanceof Error ? err.message : String(err)}\n`,
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
if (typeof override !== "object" || override === null || Array.isArray(override)) {
|
||||
console.error(`\n❌ Config override file must contain a JSON object: ${resolved}\n`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
logger.info(`${TAG} Config override loaded from: ${resolved}`);
|
||||
return deepMerge(base ?? {}, override);
|
||||
}
|
||||
|
||||
/**
|
||||
* Simple two-level deep merge: for each key in `override`, if both base
|
||||
* and override values are plain objects, merge them; otherwise override wins.
|
||||
*
|
||||
* This is sufficient for the memory-tdai config shape:
|
||||
* { capture: {...}, extraction: {...}, pipeline: {...}, ... }
|
||||
*/
|
||||
function deepMerge(
|
||||
base: Record<string, unknown>,
|
||||
override: Record<string, unknown>,
|
||||
): Record<string, unknown> {
|
||||
const result: Record<string, unknown> = { ...base };
|
||||
|
||||
for (const key of Object.keys(override)) {
|
||||
const baseVal = base[key];
|
||||
const overVal = override[key];
|
||||
|
||||
if (isPlainObject(baseVal) && isPlainObject(overVal)) {
|
||||
result[key] = { ...baseVal, ...overVal };
|
||||
} else {
|
||||
result[key] = overVal;
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
function isPlainObject(v: unknown): v is Record<string, unknown> {
|
||||
return v !== null && typeof v === "object" && !Array.isArray(v);
|
||||
}
|
||||
|
||||
function resolveOutputDir(explicit: string | undefined, stateDir: string): string {
|
||||
if (explicit) return path.resolve(explicit);
|
||||
|
||||
// Default: <stateDir>/memory-tdai-seed-<YYYYMMDD-HHmmss>
|
||||
const now = new Date();
|
||||
const pad = (n: number) => String(n).padStart(2, "0");
|
||||
const ts =
|
||||
`${now.getFullYear()}${pad(now.getMonth() + 1)}${pad(now.getDate())}-` +
|
||||
`${pad(now.getHours())}${pad(now.getMinutes())}${pad(now.getSeconds())}`;
|
||||
|
||||
return path.join(stateDir, `memory-tdai-seed-${ts}`);
|
||||
}
|
||||
|
||||
function askConfirmation(prompt: string): Promise<boolean> {
|
||||
return new Promise((resolve) => {
|
||||
// Delay slightly to let async plugin logs flush before showing the prompt.
|
||||
// Without this, the prompt gets buried under registration logs.
|
||||
setTimeout(() => {
|
||||
console.log("\n" + "─".repeat(60));
|
||||
const rl = readline.createInterface({
|
||||
input: process.stdin,
|
||||
output: process.stdout,
|
||||
});
|
||||
rl.question(`⚠️ ${prompt}`, (answer) => {
|
||||
rl.close();
|
||||
resolve(answer.trim().toLowerCase() === "y");
|
||||
});
|
||||
}, 2000);
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
/**
|
||||
* memory-tdai CLI entry point.
|
||||
*
|
||||
* Registers the `memory-tdai` namespace under the OpenClaw CLI and
|
||||
* wires up all subcommands (currently: `seed`).
|
||||
*
|
||||
* Integration path:
|
||||
* index.ts → api.registerCli() → registerMemoryTdaiCli() → registerSeedCommand()
|
||||
*/
|
||||
|
||||
import type { Command } from "commander";
|
||||
import { registerSeedCommand } from "./commands/seed.js";
|
||||
|
||||
// ============================
|
||||
// Context type
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Minimal context needed by seed CLI commands.
|
||||
*
|
||||
* Derived from OpenClawPluginCliContext but scoped to what seed actually needs,
|
||||
* avoiding a hard dependency on the full plugin CLI context type.
|
||||
*/
|
||||
export interface SeedCliContext {
|
||||
/** OpenClaw config (for LLM calls in L1 extraction). */
|
||||
config: unknown;
|
||||
/** Raw plugin config (same shape as api.pluginConfig). */
|
||||
pluginConfig: unknown;
|
||||
/** State directory root (e.g. ~/.openclaw). */
|
||||
stateDir: string;
|
||||
/** Logger instance. */
|
||||
logger: {
|
||||
debug?: (message: string) => void;
|
||||
info: (message: string) => void;
|
||||
warn: (message: string) => void;
|
||||
error: (message: string) => void;
|
||||
};
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Top-level registration
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Register all memory-tdai CLI subcommands under the given Commander program.
|
||||
*
|
||||
* This function is called by the plugin's `api.registerCli()` registrar.
|
||||
* It creates the `memory-tdai` namespace and delegates to individual
|
||||
* command registrars.
|
||||
*
|
||||
* @param program - The `memory-tdai` Commander command (already created by the registrar)
|
||||
* @param ctx - CLI context with config, state dir, and logger
|
||||
*/
|
||||
export function registerMemoryTdaiCli(program: Command, ctx: SeedCliContext): void {
|
||||
// Register subcommands
|
||||
registerSeedCommand(program, ctx);
|
||||
|
||||
// Future: registerQueryCommand(program, ctx);
|
||||
// Future: registerStatsCommand(program, ctx);
|
||||
}
|
||||
Reference in New Issue
Block a user