mirror of
https://github.com/TencentCloud/TencentDB-Agent-Memory
synced 2026-07-11 04:44:29 +00:00
feat: release v0.3.3 — Hermes adapter, context offload, core refactor
This commit is contained in:
chrishuan
@@ -0,0 +1,582 @@
|
||||
/**
|
||||
* L0 Conversation Recorder: records raw conversation messages to local JSONL files.
|
||||
*
|
||||
* Triggered from agent_end hook. Receives the conversation messages directly from
|
||||
* the hook context (no file I/O needed), sanitizes them, filters out noise, and
|
||||
* writes to ~/.openclaw/memory-tdai/conversations/YYYY-MM-DD.jsonl
|
||||
*
|
||||
* Design decisions:
|
||||
* - Uses JSONL format (**one message per line** — flat, easy to grep/stream)
|
||||
* - One file per day (all sessions merged into the same daily file)
|
||||
* - sessionKey is stored as a field in each JSONL line, not in the filename
|
||||
* - Independent from system session files — format fully controlled by plugin
|
||||
* - Messages are sanitized to remove injected tags (prevent feedback loops)
|
||||
* - Short/long/command messages are filtered out
|
||||
*/
|
||||
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import crypto from "node:crypto";
|
||||
import { sanitizeText, stripCodeBlocks, shouldCaptureL0 } from "../../utils/sanitize.js";
|
||||
|
||||
// ============================
|
||||
// Types
|
||||
// ============================
|
||||
|
||||
export interface ConversationMessage {
|
||||
/** Unique message ID (used by L1 prompt for source_message_ids tracking) */
|
||||
id: string;
|
||||
role: "user" | "assistant";
|
||||
content: string;
|
||||
timestamp: number; // epoch ms
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a short unique message ID.
|
||||
*/
|
||||
function generateMessageId(): string {
|
||||
return `msg_${Date.now()}_${crypto.randomBytes(3).toString("hex")}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* New flat format: one message per JSONL line.
|
||||
*/
|
||||
export interface L0MessageRecord {
|
||||
sessionKey: string;
|
||||
sessionId: string;
|
||||
recordedAt: string; // ISO timestamp
|
||||
id: string;
|
||||
role: "user" | "assistant";
|
||||
content: string;
|
||||
timestamp: number; // epoch ms
|
||||
}
|
||||
|
||||
/**
|
||||
* A group of conversation messages (used by downstream consumers).
|
||||
* Each L0ConversationRecord represents one or more messages from the same recording event.
|
||||
*/
|
||||
export interface L0ConversationRecord {
|
||||
sessionKey: string;
|
||||
sessionId: string;
|
||||
recordedAt: string; // ISO timestamp
|
||||
messageCount: number;
|
||||
messages: ConversationMessage[];
|
||||
}
|
||||
|
||||
interface Logger {
|
||||
debug?: (message: string) => void;
|
||||
info: (message: string) => void;
|
||||
warn: (message: string) => void;
|
||||
error: (message: string) => void;
|
||||
}
|
||||
|
||||
const TAG = "[memory-tdai][l0]";
|
||||
|
||||
// ============================
|
||||
// Core function
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Record a conversation round to the L0 JSONL file.
|
||||
*
|
||||
* Only records **incremental** messages (new since the last capture).
|
||||
* Uses `afterTimestamp` as the primary filter to skip already-captured history.
|
||||
*
|
||||
* @param sessionKey - The session key for this conversation
|
||||
* @param rawMessages - Raw messages from the agent_end hook context (full session history)
|
||||
* @param baseDir - Base data directory (~/.openclaw/memory-tdai/)
|
||||
* @param logger - Optional logger
|
||||
* @param originalUserText - Clean original user prompt (pre-prependContext)
|
||||
* @param afterTimestamp - Epoch ms cursor: only messages with timestamp > this are new.
|
||||
* Pass 0 or omit for the first capture of a session.
|
||||
* @returns Filtered messages (for L1 to use directly), or empty array if nothing worth recording
|
||||
*/
|
||||
export async function recordConversation(params: {
|
||||
sessionKey: string;
|
||||
sessionId?: string;
|
||||
rawMessages: unknown[];
|
||||
baseDir: string;
|
||||
logger?: Logger;
|
||||
/** Clean original user prompt (pre-prependContext) */
|
||||
originalUserText?: string;
|
||||
/** Epoch ms cursor: only process messages with timestamp strictly greater than this. */
|
||||
afterTimestamp?: number;
|
||||
/**
|
||||
* Number of messages in the session at before_prompt_build time.
|
||||
* Used to locate the exact user message that originalUserText corresponds to:
|
||||
* rawMessages[originalUserMessageCount] is the user message appended by the framework
|
||||
* AFTER before_prompt_build, i.e. the one whose content was polluted by prependContext.
|
||||
*/
|
||||
originalUserMessageCount?: number;
|
||||
}): Promise<ConversationMessage[]> {
|
||||
const { sessionKey, sessionId, rawMessages, baseDir, logger, originalUserText, afterTimestamp, originalUserMessageCount } = params;
|
||||
|
||||
// Step 1: Position slice + extract user/assistant messages.
|
||||
//
|
||||
// Dual protection against duplicate capture:
|
||||
// Layer 1 (position slice): Use originalUserMessageCount (cached at before_prompt_build)
|
||||
// to slice rawMessages — only keep messages added AFTER the prompt build, i.e. this
|
||||
// turn's new messages. This is immune to timestamp drift after gateway restarts.
|
||||
// Layer 2 (timestamp cursor): The existing afterTimestamp filter below acts as a fallback
|
||||
// when the position slice is unavailable (cache expired, process restart, etc.).
|
||||
const usePositionSlice = originalUserMessageCount != null && originalUserMessageCount > 0
|
||||
&& originalUserMessageCount <= rawMessages.length;
|
||||
const slicedMessages = usePositionSlice
|
||||
? rawMessages.slice(originalUserMessageCount)
|
||||
: rawMessages;
|
||||
|
||||
const allExtracted = extractUserAssistantMessages(slicedMessages);
|
||||
|
||||
if (usePositionSlice) {
|
||||
logger?.debug?.(
|
||||
`${TAG} Position slice: ${rawMessages.length} raw → ${slicedMessages.length} new (sliceStart=${originalUserMessageCount})`,
|
||||
);
|
||||
}
|
||||
|
||||
// Diagnostic: check whether the framework actually provides timestamp on raw messages.
|
||||
// If all raw timestamps are missing, the timestamp cursor is effectively useless and
|
||||
// position slice becomes the sole incremental mechanism.
|
||||
if (slicedMessages.length > 0) {
|
||||
const firstRaw = slicedMessages[0] as Record<string, unknown> | undefined;
|
||||
const rawTs = firstRaw?.timestamp;
|
||||
const hasRawTs = typeof rawTs === "number";
|
||||
logger?.debug?.(
|
||||
`${TAG} Raw message[0] timestamp probe: ${hasRawTs ? `present (${rawTs})` : `missing (type=${typeof rawTs}, value=${String(rawTs)})`}`,
|
||||
);
|
||||
}
|
||||
|
||||
logger?.debug?.(`${TAG} Extracted ${allExtracted.length} user/assistant messages from ${slicedMessages.length} total`);
|
||||
|
||||
// Step 1.5: Incremental filter — only keep messages newer than the cursor.
|
||||
//
|
||||
// Uses strict greater-than (>) which is safe because:
|
||||
// - The cursor is set to max(timestamps) of the LAST recorded batch.
|
||||
// - The next agent turn's messages will have timestamps strictly greater than
|
||||
// the previous turn (there's at least one LLM API call between turns, which
|
||||
// takes hundreds of milliseconds minimum — no same-millisecond collision).
|
||||
// - All messages within a single turn are captured together as one batch,
|
||||
// so even if multiple messages share the same timestamp, they are either
|
||||
// all included (new batch) or all excluded (already captured).
|
||||
// - If a message lacks a timestamp field, extractUserAssistantMessages()
|
||||
// assigns Date.now() at extraction time, which is always > previous cursor.
|
||||
const cursor = afterTimestamp ?? 0;
|
||||
const extracted = cursor !== 0
|
||||
? allExtracted.filter((m) => m.timestamp > cursor)
|
||||
: allExtracted;
|
||||
|
||||
if (extracted.length > 0) {
|
||||
const first = extracted[0];
|
||||
logger?.debug?.(
|
||||
`${TAG} First captured message: role=${first.role}, ts=${first.timestamp}, ` +
|
||||
`date=${new Date(first.timestamp).toISOString()}, content=${first.content.slice(0, 80)}${first.content.length > 80 ? "…" : ""}`,
|
||||
);
|
||||
}
|
||||
|
||||
if (cursor > 0) {
|
||||
logger?.debug?.(
|
||||
`${TAG} Incremental filter: ${allExtracted.length} total → ${extracted.length} new (cursor=${cursor})`,
|
||||
);
|
||||
|
||||
// Safety valve: if timestamp filter passed everything through and position slice
|
||||
// was not available, this likely indicates timestamp drift after a gateway restart.
|
||||
if (!usePositionSlice && extracted.length === allExtracted.length && allExtracted.length > 8) {
|
||||
logger?.warn?.(
|
||||
`${TAG} ⚠ Safety valve: all ${allExtracted.length} messages passed timestamp filter (cursor=${cursor}) — ` +
|
||||
`possible timestamp drift after gateway restart. Position slice was not available (no cached messageCount).`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (extracted.length === 0) {
|
||||
logger?.debug?.(`${TAG} No new user/assistant messages to record`);
|
||||
return [];
|
||||
}
|
||||
|
||||
// Step 2: Replace polluted user messages with cached original prompt.
|
||||
//
|
||||
// Background:
|
||||
// The framework appends the user's message to the session after before_prompt_build,
|
||||
// then injects prependContext into it. So the user message in rawMessages is polluted.
|
||||
// We cached the clean prompt (originalUserText) and the message count at
|
||||
// before_prompt_build time (originalUserMessageCount) to identify which raw message
|
||||
// is the real user input.
|
||||
//
|
||||
// Strategy:
|
||||
// When position slice is active, the polluted user message is slicedMessages[0].
|
||||
// Otherwise, fall back to rawMessages[originalUserMessageCount].
|
||||
// In both cases, find the timestamp and match it in `extracted` for replacement.
|
||||
// If matching fails, skip replacement — sanitizeText() in Step 3 is the safety net.
|
||||
if (originalUserText) {
|
||||
// Determine the target raw message that contains the polluted user prompt
|
||||
const targetRaw: Record<string, unknown> | undefined = usePositionSlice
|
||||
? slicedMessages[0] as Record<string, unknown> | undefined
|
||||
: (originalUserMessageCount != null && originalUserMessageCount >= 0 && originalUserMessageCount < rawMessages.length)
|
||||
? rawMessages[originalUserMessageCount] as Record<string, unknown> | undefined
|
||||
: undefined;
|
||||
|
||||
const targetTs = targetRaw && typeof targetRaw.timestamp === "number" ? targetRaw.timestamp : undefined;
|
||||
|
||||
if (targetTs != null) {
|
||||
let replaced = false;
|
||||
for (let i = 0; i < extracted.length; i++) {
|
||||
if (extracted[i].role === "user" && extracted[i].timestamp === targetTs) {
|
||||
logger?.debug?.(
|
||||
`${TAG} Replacing user message at timestamp=${targetTs} with cached original prompt ` +
|
||||
`(${originalUserText.length} chars, was ${extracted[i].content.length} chars) [positionSlice=${usePositionSlice}]`,
|
||||
);
|
||||
extracted[i] = { ...extracted[i], content: originalUserText };
|
||||
replaced = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!replaced) {
|
||||
logger?.warn?.(
|
||||
`${TAG} Target user message (ts=${targetTs}) not found in extracted batch — ` +
|
||||
`possibly filtered by cursor. Skipping replacement, will rely on sanitizeText().`,
|
||||
);
|
||||
}
|
||||
} else if (targetRaw) {
|
||||
logger?.warn?.(
|
||||
`${TAG} Target raw message has no valid timestamp — ` +
|
||||
`skipping replacement, will rely on sanitizeText().`,
|
||||
);
|
||||
} else {
|
||||
logger?.warn?.(
|
||||
`${TAG} Have originalUserText but cannot locate target raw message — ` +
|
||||
`skipping replacement, will rely on sanitizeText().`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Step 3: Sanitize and filter
|
||||
const filtered = extracted
|
||||
.map((m) => {
|
||||
let content = sanitizeText(m.content);
|
||||
// Strip fenced code blocks from assistant replies to reduce embedding noise
|
||||
if (m.role === "assistant") {
|
||||
content = stripCodeBlocks(content);
|
||||
}
|
||||
return { id: m.id, role: m.role, content, timestamp: m.timestamp };
|
||||
})
|
||||
.filter((m) => shouldCaptureL0(m.content));
|
||||
|
||||
logger?.debug?.(`${TAG} After sanitize+filter: ${filtered.length} messages (from ${extracted.length})`);
|
||||
|
||||
if (filtered.length === 0) {
|
||||
logger?.debug?.(`${TAG} All messages filtered out, skipping L0 write`);
|
||||
return [];
|
||||
}
|
||||
|
||||
// Step 4: Write to JSONL file — one message per line (flat format)
|
||||
const now = new Date().toISOString();
|
||||
const lines: string[] = [];
|
||||
for (const msg of filtered) {
|
||||
const record: L0MessageRecord = {
|
||||
sessionKey,
|
||||
sessionId: sessionId || "",
|
||||
recordedAt: now,
|
||||
id: msg.id,
|
||||
role: msg.role,
|
||||
content: msg.content,
|
||||
timestamp: msg.timestamp,
|
||||
};
|
||||
lines.push(JSON.stringify(record));
|
||||
}
|
||||
|
||||
const shardDate = formatLocalDate(new Date());
|
||||
const outDir = path.join(baseDir, "conversations");
|
||||
const outPath = path.join(outDir, `${shardDate}.jsonl`);
|
||||
|
||||
try {
|
||||
await fs.mkdir(outDir, { recursive: true });
|
||||
// Append each message as its own JSONL line
|
||||
await fs.appendFile(outPath, lines.join("\n") + "\n", "utf-8");
|
||||
logger?.debug?.(`${TAG} Recorded ${filtered.length} messages to ${outPath}`);
|
||||
} catch (err) {
|
||||
logger?.error(`${TAG} Failed to write L0 file: ${err instanceof Error ? err.message : String(err)}`);
|
||||
// Return filtered messages anyway so L1 can still process them
|
||||
}
|
||||
|
||||
return filtered;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read all L0 conversation records for a session.
|
||||
* Returns records in chronological order.
|
||||
*
|
||||
* File format: `YYYY-MM-DD.jsonl` (daily files, all sessions merged).
|
||||
* Each line is an L0MessageRecord; filtered by sessionKey at line level.
|
||||
*/
|
||||
export async function readConversationRecords(
|
||||
sessionKey: string,
|
||||
baseDir: string,
|
||||
logger?: Logger,
|
||||
): Promise<L0ConversationRecord[]> {
|
||||
const conversationsDir = path.join(baseDir, "conversations");
|
||||
|
||||
// Daily file pattern: YYYY-MM-DD.jsonl
|
||||
const dateFilePattern = /^\d{4}-\d{2}-\d{2}\.jsonl$/;
|
||||
|
||||
let entries: string[];
|
||||
try {
|
||||
const dirEntries = await fs.readdir(conversationsDir, { withFileTypes: true });
|
||||
entries = dirEntries
|
||||
.filter((entry) => entry.isFile())
|
||||
.map((entry) => entry.name);
|
||||
} catch {
|
||||
// Directory doesn't exist yet — normal for first conversation
|
||||
return [];
|
||||
}
|
||||
|
||||
const targetFiles = entries
|
||||
.filter((name) => dateFilePattern.test(name))
|
||||
.sort();
|
||||
|
||||
if (targetFiles.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const records: L0ConversationRecord[] = [];
|
||||
|
||||
for (const fileName of targetFiles) {
|
||||
const filePath = path.join(conversationsDir, fileName);
|
||||
|
||||
let raw: string;
|
||||
try {
|
||||
raw = await fs.readFile(filePath, "utf-8");
|
||||
} catch {
|
||||
logger?.warn?.(`${TAG} Failed to read L0 file: ${filePath}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const lines = raw.split("\n").filter((line: string) => line.trim());
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const line = lines[i];
|
||||
try {
|
||||
const parsed = JSON.parse(line) as Record<string, unknown>;
|
||||
|
||||
// Filter by sessionKey at line level
|
||||
const lineSessionKey = parsed.sessionKey as string | undefined;
|
||||
if (lineSessionKey !== sessionKey) continue;
|
||||
|
||||
if (typeof parsed.role === "string" && typeof parsed.content === "string") {
|
||||
// Flat format: { sessionKey, sessionId, recordedAt, id, role, content, timestamp }
|
||||
// Wrap into L0ConversationRecord for uniform downstream consumption
|
||||
const msg: ConversationMessage = {
|
||||
id: (typeof parsed.id === "string" && parsed.id) ? parsed.id : generateMessageId(),
|
||||
role: parsed.role as "user" | "assistant",
|
||||
content: parsed.content as string,
|
||||
timestamp: typeof parsed.timestamp === "number" ? parsed.timestamp : Date.now(),
|
||||
};
|
||||
records.push({
|
||||
sessionKey: (parsed.sessionKey as string) || sessionKey,
|
||||
sessionId: (parsed.sessionId as string) || "",
|
||||
recordedAt: (parsed.recordedAt as string) || new Date().toISOString(),
|
||||
messageCount: 1,
|
||||
messages: [msg],
|
||||
});
|
||||
} else {
|
||||
logger?.warn?.(`${TAG} Unrecognized JSONL line format in ${filePath}:${i + 1}`);
|
||||
}
|
||||
} catch {
|
||||
logger?.warn?.(`${TAG} Skipping malformed JSONL line in ${filePath}:${i + 1}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
records.sort((a, b) => {
|
||||
const ta = Date.parse(a.recordedAt);
|
||||
const tb = Date.parse(b.recordedAt);
|
||||
const na = Number.isFinite(ta) ? ta : Number.POSITIVE_INFINITY;
|
||||
const nb = Number.isFinite(tb) ? tb : Number.POSITIVE_INFINITY;
|
||||
return na - nb;
|
||||
});
|
||||
|
||||
return records;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read L0 messages across all conversation records for a session,
|
||||
* optionally filtered by a cursor timestamp (messages after the cursor).
|
||||
*
|
||||
* When `limit` is provided, only the **newest** `limit` messages are returned
|
||||
* (matching the DB path's `ORDER BY timestamp DESC LIMIT ?` behavior).
|
||||
* Returned messages are always in chronological order (oldest → newest).
|
||||
*
|
||||
* NOTE: potential optimization — records are chronologically ordered (append-only JSONL),
|
||||
* so a reverse scan could skip entire old records. Deferred for now; see Issue 5 in
|
||||
* docs/05-known-issues.md.
|
||||
*/
|
||||
export async function readConversationMessages(
|
||||
sessionKey: string,
|
||||
baseDir: string,
|
||||
afterTimestamp?: number,
|
||||
logger?: Logger,
|
||||
limit?: number,
|
||||
): Promise<ConversationMessage[]> {
|
||||
const records = await readConversationRecords(sessionKey, baseDir, logger);
|
||||
const allMessages: ConversationMessage[] = [];
|
||||
|
||||
for (const record of records) {
|
||||
for (const msg of record.messages) {
|
||||
if (afterTimestamp && msg.timestamp <= afterTimestamp) continue;
|
||||
allMessages.push(msg);
|
||||
}
|
||||
}
|
||||
|
||||
// Truncate to newest `limit` messages (keep tail, since array is chronological)
|
||||
if (limit != null && limit > 0 && allMessages.length > limit) {
|
||||
logger?.debug?.(
|
||||
`${TAG} readConversationMessages: truncating ${allMessages.length} → ${limit} (newest)`,
|
||||
);
|
||||
return allMessages.slice(-limit);
|
||||
}
|
||||
|
||||
return allMessages;
|
||||
}
|
||||
|
||||
/**
|
||||
* A group of conversation messages sharing the same sessionId.
|
||||
*/
|
||||
export interface SessionIdMessageGroup {
|
||||
sessionId: string;
|
||||
messages: Array<ConversationMessage & { recordedAtMs: number }>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read L0 messages for a session, grouped by sessionId.
|
||||
*
|
||||
* Within the same sessionKey, different sessionIds represent different conversation
|
||||
* instances (e.g. after /reset). L1 extraction should process each group independently
|
||||
* so that each group's sessionId is correctly associated with its extracted memories.
|
||||
*
|
||||
* When `limit` is provided, only the **newest** `limit` messages (across all groups)
|
||||
* are retained — matching the DB path's `ORDER BY recorded_at DESC LIMIT ?` behavior.
|
||||
* Groups that become empty after truncation are dropped.
|
||||
*
|
||||
* Groups are returned in chronological order (by earliest message timestamp).
|
||||
* Messages within each group are also in chronological order.
|
||||
*
|
||||
* @param afterRecordedAtMs - Epoch ms cursor: only messages with recordedAt > this are included.
|
||||
*/
|
||||
export async function readConversationMessagesGroupedBySessionId(
|
||||
sessionKey: string,
|
||||
baseDir: string,
|
||||
afterRecordedAtMs?: number,
|
||||
logger?: Logger,
|
||||
limit?: number,
|
||||
): Promise<SessionIdMessageGroup[]> {
|
||||
const records = await readConversationRecords(sessionKey, baseDir, logger);
|
||||
|
||||
// Collect all messages with their sessionId, filtering by recorded_at cursor
|
||||
const allMessages: Array<{ sessionId: string; msg: ConversationMessage & { recordedAtMs: number } }> = [];
|
||||
|
||||
for (const record of records) {
|
||||
const sid = record.sessionId || "";
|
||||
const recMs = Date.parse(record.recordedAt) || 0;
|
||||
if (afterRecordedAtMs && recMs <= afterRecordedAtMs) continue;
|
||||
for (const msg of record.messages) {
|
||||
allMessages.push({ sessionId: sid, msg: { ...msg, recordedAtMs: recMs } });
|
||||
}
|
||||
}
|
||||
|
||||
// Sort by timestamp ASC (chronological) — records are already roughly ordered
|
||||
// by recordedAt, but messages within may not be perfectly sorted by timestamp.
|
||||
allMessages.sort((a, b) => a.msg.timestamp - b.msg.timestamp);
|
||||
|
||||
// Truncate to newest `limit` messages (keep tail)
|
||||
let selected = allMessages;
|
||||
if (limit != null && limit > 0 && allMessages.length > limit) {
|
||||
logger?.debug?.(
|
||||
`${TAG} readConversationMessagesGroupedBySessionId: truncating ${allMessages.length} → ${limit} (newest)`,
|
||||
);
|
||||
selected = allMessages.slice(-limit);
|
||||
}
|
||||
|
||||
// Re-group by sessionId
|
||||
const groupMap = new Map<string, Array<ConversationMessage & { recordedAtMs: number }>>();
|
||||
for (const { sessionId, msg } of selected) {
|
||||
let group = groupMap.get(sessionId);
|
||||
if (!group) {
|
||||
group = [];
|
||||
groupMap.set(sessionId, group);
|
||||
}
|
||||
group.push(msg);
|
||||
}
|
||||
|
||||
// Convert to array, sorted by earliest message timestamp in each group
|
||||
const groups: SessionIdMessageGroup[] = [];
|
||||
for (const [sessionId, messages] of groupMap) {
|
||||
if (messages.length > 0) {
|
||||
groups.push({ sessionId, messages });
|
||||
}
|
||||
}
|
||||
groups.sort((a, b) => a.messages[0].timestamp - b.messages[0].timestamp);
|
||||
|
||||
return groups;
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Helpers
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Extract user and assistant messages from raw hook message array.
|
||||
*/
|
||||
function extractUserAssistantMessages(messages: unknown[]): ConversationMessage[] {
|
||||
const result: ConversationMessage[] = [];
|
||||
|
||||
for (const msg of messages) {
|
||||
if (!msg || typeof msg !== "object") continue;
|
||||
const m = msg as Record<string, unknown>;
|
||||
const role = m.role as string | undefined;
|
||||
|
||||
if (role !== "user" && role !== "assistant") continue;
|
||||
|
||||
let content: string | undefined;
|
||||
if (typeof m.content === "string") {
|
||||
content = m.content;
|
||||
} else if (Array.isArray(m.content)) {
|
||||
const textParts: string[] = [];
|
||||
for (const part of m.content) {
|
||||
if (
|
||||
part &&
|
||||
typeof part === "object" &&
|
||||
(part as Record<string, unknown>).type === "text"
|
||||
) {
|
||||
const text = (part as Record<string, unknown>).text;
|
||||
if (typeof text === "string") textParts.push(text);
|
||||
}
|
||||
}
|
||||
content = textParts.join("\n");
|
||||
}
|
||||
|
||||
// Strip inline base64 image data URIs that some providers embed in string content.
|
||||
// These are not useful for memory and would pollute FTS / embedding indexes.
|
||||
if (content && /data:image\/[a-z+]+;base64,/i.test(content)) {
|
||||
content = content.replace(/data:image\/[a-z+]+;base64,[A-Za-z0-9+/=]+/gi, "[image]");
|
||||
}
|
||||
|
||||
if (content && content.trim()) {
|
||||
const ts = typeof m.timestamp === "number" ? m.timestamp : Date.now();
|
||||
result.push({
|
||||
id: (typeof m.id === "string" && m.id) ? m.id : generateMessageId(),
|
||||
role: role as "user" | "assistant",
|
||||
content: content.trim(),
|
||||
timestamp: ts,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Format local date as YYYY-MM-DD.
|
||||
*/
|
||||
function formatLocalDate(d: Date): string {
|
||||
const y = d.getFullYear();
|
||||
const m = String(d.getMonth() + 1).padStart(2, "0");
|
||||
const day = String(d.getDate()).padStart(2, "0");
|
||||
return `${y}-${m}-${day}`;
|
||||
}
|
||||
@@ -0,0 +1,348 @@
|
||||
/**
|
||||
* auto-capture hook (v3): records conversation messages locally (L0),
|
||||
* then notifies the MemoryPipelineManager for L1/L2/L3 scheduling.
|
||||
*
|
||||
* Key design decisions:
|
||||
* - Always write L0 locally via l0-recorder.
|
||||
* - When VectorStore + EmbeddingService are available, also write L0 vector index.
|
||||
* - Notify MemoryPipelineManager for L1/L2/L3 trigger evaluation.
|
||||
* - L1 Runner reads from VectorStore DB (primary) or L0 JSONL files (fallback).
|
||||
* - Extraction is NOT triggered here. The pipeline manager decides when.
|
||||
*/
|
||||
|
||||
import crypto from "node:crypto";
|
||||
import type { MemoryTdaiConfig } from "../../config.js";
|
||||
import { CheckpointManager } from "../../utils/checkpoint.js";
|
||||
import type { MemoryPipelineManager } from "../../utils/pipeline-manager.js";
|
||||
import { recordConversation } from "../conversation/l0-recorder.js";
|
||||
import type { ConversationMessage } from "../conversation/l0-recorder.js";
|
||||
import type { IMemoryStore, L0Record } from "../store/types.js";
|
||||
import type { EmbeddingService } from "../store/embedding.js";
|
||||
|
||||
const TAG = "[memory-tdai] [capture]";
|
||||
|
||||
interface Logger {
|
||||
debug?: (message: string) => void;
|
||||
info: (message: string) => void;
|
||||
warn: (message: string) => void;
|
||||
error: (message: string) => void;
|
||||
}
|
||||
|
||||
export interface AutoCaptureResult {
|
||||
/** Whether the scheduler was notified (conversation count incremented) */
|
||||
schedulerNotified: boolean;
|
||||
/** Number of messages recorded to L0 */
|
||||
l0RecordedCount: number;
|
||||
/** Number of L0 message vectors written */
|
||||
l0VectorsWritten: number;
|
||||
/** Filtered messages for L1 immediate use */
|
||||
filteredMessages: ConversationMessage[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a unique L0 record ID for vector indexing.
|
||||
* Includes an index to distinguish multiple messages within the same round.
|
||||
*/
|
||||
function generateL0RecordId(sessionKey: string, index: number): string {
|
||||
return `l0_${sessionKey}_${Date.now()}_${index}_${crypto.randomBytes(3).toString("hex")}`;
|
||||
}
|
||||
|
||||
export async function performAutoCapture(params: {
|
||||
messages: unknown[];
|
||||
sessionKey: string;
|
||||
sessionId?: string;
|
||||
cfg: MemoryTdaiConfig;
|
||||
pluginDataDir: string;
|
||||
logger?: Logger;
|
||||
scheduler?: MemoryPipelineManager;
|
||||
/** Clean original user prompt from before_prompt_build cache (pre-prependContext). */
|
||||
originalUserText?: string;
|
||||
/**
|
||||
* Number of messages in the session at before_prompt_build time.
|
||||
* Used by l0-recorder to locate the exact user message that originalUserText
|
||||
* corresponds to: rawMessages[originalUserMessageCount] is the polluted user message.
|
||||
*/
|
||||
originalUserMessageCount?: number;
|
||||
/** Epoch ms when the plugin was registered (cold-start time).
|
||||
* Used as fallback cursor when checkpoint has no prior timestamp —
|
||||
* prevents the first agent_end from dumping all session history into L0. */
|
||||
pluginStartTimestamp?: number;
|
||||
/** VectorStore for L0 vector indexing (optional). */
|
||||
vectorStore?: IMemoryStore;
|
||||
/** EmbeddingService for L0 vector indexing (optional). */
|
||||
embeddingService?: EmbeddingService;
|
||||
/**
|
||||
* Tracks in-flight fire-and-forget background tasks started by this
|
||||
* capture (currently: deferred L0 embedding for SQLite-style stores).
|
||||
*
|
||||
* When provided, each background task's Promise is added to the set
|
||||
* on creation and removed on completion. This lets the owning
|
||||
* ``TdaiCore`` instance await all pending background work before
|
||||
* closing ``vectorStore`` / ``embeddingService`` in ``destroy()``,
|
||||
* so we never hit an already-closed DB connection with a late
|
||||
* ``updateL0Embedding`` call.
|
||||
*
|
||||
* Optional for backwards compatibility — callers that don't care
|
||||
* (tests, short-lived CLI invocations) can omit it and accept the
|
||||
* pre-fix behaviour (background task may outlive its owner).
|
||||
*/
|
||||
bgTaskRegistry?: Set<Promise<void>>;
|
||||
}): Promise<AutoCaptureResult> {
|
||||
const {
|
||||
messages, sessionKey, sessionId, cfg, pluginDataDir, logger, scheduler,
|
||||
originalUserText, originalUserMessageCount, pluginStartTimestamp,
|
||||
vectorStore, embeddingService, bgTaskRegistry,
|
||||
} = params;
|
||||
const tCaptureStart = performance.now();
|
||||
|
||||
const checkpoint = new CheckpointManager(pluginDataDir, logger);
|
||||
|
||||
// ============================
|
||||
// Step 1 + 2: L0 recording + checkpoint update (ATOMIC)
|
||||
// ============================
|
||||
// These steps are combined inside captureAtomically() to prevent the race
|
||||
// condition where two concurrent agent_end events both read the same stale
|
||||
// cursor and produce duplicate L0 records. The file lock is held for the
|
||||
// entire read-cursor → recordConversation → advance-cursor sequence.
|
||||
const tL0RecordStart = performance.now();
|
||||
let filteredMessages: ConversationMessage[] = [];
|
||||
try {
|
||||
await checkpoint.captureAtomically(
|
||||
sessionKey,
|
||||
pluginStartTimestamp,
|
||||
async (afterTimestamp) => {
|
||||
logger?.debug?.(`${TAG} L0 capture cursor (per-session, atomic): afterTimestamp=${afterTimestamp} session=${sessionKey}`);
|
||||
|
||||
if (afterTimestamp === pluginStartTimestamp && pluginStartTimestamp && pluginStartTimestamp > 0) {
|
||||
logger?.debug?.(
|
||||
`${TAG} No per-session checkpoint cursor found for session=${sessionKey} — ` +
|
||||
`using pluginStartTimestamp as floor: ` +
|
||||
`${afterTimestamp} (${new Date(afterTimestamp).toISOString()})`,
|
||||
);
|
||||
}
|
||||
|
||||
filteredMessages = await recordConversation({
|
||||
sessionKey,
|
||||
sessionId,
|
||||
rawMessages: messages,
|
||||
baseDir: pluginDataDir,
|
||||
logger,
|
||||
originalUserText,
|
||||
afterTimestamp,
|
||||
originalUserMessageCount,
|
||||
});
|
||||
|
||||
if (filteredMessages.length === 0) {
|
||||
return null; // Nothing captured — cursor stays unchanged
|
||||
}
|
||||
|
||||
logger?.debug?.(`${TAG} L0 recorded: ${filteredMessages.length} messages for session ${sessionKey}`);
|
||||
const maxTs = Math.max(...filteredMessages.map((m) => m.timestamp));
|
||||
return { maxTimestamp: maxTs, messageCount: filteredMessages.length };
|
||||
},
|
||||
);
|
||||
} catch (err) {
|
||||
logger?.error(`${TAG} L0 recording failed: ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
const tL0RecordEnd = performance.now();
|
||||
|
||||
// ============================
|
||||
// Step 1.5: L0 vector indexing
|
||||
// ============================
|
||||
// Two paths depending on store capabilities:
|
||||
//
|
||||
// A) Store supports updateL0Embedding (sqlite):
|
||||
// - Write metadata + FTS immediately WITHOUT embedding (~ms)
|
||||
// - Fire-and-forget background task: embedBatch + updateL0Embedding
|
||||
// - PERF: avoids blocking agent_end with 2-3s embedding calls
|
||||
//
|
||||
// B) Store does NOT support updateL0Embedding (VDB / remote):
|
||||
// - Embed synchronously, then upsertL0 with embedding in one call
|
||||
// - VDB backends handle embedding server-side or need it upfront
|
||||
const tL0VecStart = performance.now();
|
||||
let l0VectorsWritten = 0;
|
||||
let l0EmbedTotalMs = 0;
|
||||
let l0UpsertTotalMs = 0;
|
||||
logger?.debug?.(
|
||||
`${TAG} [L0-vec-index] Check: filteredMessages=${filteredMessages.length}, ` +
|
||||
`vectorStore=${vectorStore ? "available" : "UNAVAILABLE"}, ` +
|
||||
`embeddingService=${embeddingService ? "available" : "UNAVAILABLE"}`,
|
||||
);
|
||||
|
||||
const supportsBgEmbed = vectorStore?.supportsDeferredEmbedding === true;
|
||||
|
||||
if (filteredMessages.length > 0 && vectorStore) {
|
||||
const now = new Date().toISOString();
|
||||
const bgRecords: Array<{ recordId: string; content: string }> = [];
|
||||
logger?.debug?.(
|
||||
`${TAG} [L0-vec-index] START indexing ${filteredMessages.length} message(s) for session ${sessionKey} ` +
|
||||
`(mode=${supportsBgEmbed ? "async-bg" : "sync"})`,
|
||||
);
|
||||
|
||||
for (let i = 0; i < filteredMessages.length; i++) {
|
||||
const msg = filteredMessages[i];
|
||||
try {
|
||||
const l0Record: L0Record = {
|
||||
id: generateL0RecordId(sessionKey, i),
|
||||
sessionKey,
|
||||
sessionId: sessionId || "",
|
||||
role: msg.role,
|
||||
messageText: msg.content,
|
||||
recordedAt: now,
|
||||
timestamp: msg.timestamp,
|
||||
};
|
||||
|
||||
let embedding: Float32Array | undefined;
|
||||
|
||||
if (!supportsBgEmbed && embeddingService) {
|
||||
// Path B (VDB): embed synchronously — needed for upsertL0
|
||||
// Skip local embed when using server-side embedding (NoopEmbeddingService, dims=0)
|
||||
if (embeddingService.getDimensions() === 0) {
|
||||
logger?.debug?.(
|
||||
`${TAG} [L0-vec-index] Server-side embedding (dims=0), skipping local embed for message ${i}`,
|
||||
);
|
||||
} else {
|
||||
const tEmbedStart = performance.now();
|
||||
try {
|
||||
embedding = await embeddingService.embed(msg.content);
|
||||
l0EmbedTotalMs += performance.now() - tEmbedStart;
|
||||
logger?.debug?.(
|
||||
`${TAG} [L0-vec-index] Embedding OK: dims=${embedding.length}, ` +
|
||||
`norm=${Math.sqrt(Array.from(embedding).reduce((s, v) => s + v * v, 0)).toFixed(4)}`,
|
||||
);
|
||||
} catch (embedErr) {
|
||||
l0EmbedTotalMs += performance.now() - tEmbedStart;
|
||||
logger?.warn(
|
||||
`${TAG} [L0-vec-index] Embedding FAILED for message ${i}, ` +
|
||||
`will write metadata only: ${embedErr instanceof Error ? embedErr.message : String(embedErr)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Path A (sqlite): pass undefined embedding — metadata + FTS only
|
||||
// Path B (VDB): pass embedding (may be undefined on failure)
|
||||
const tUpsertStart = performance.now();
|
||||
const upsertOk = await vectorStore.upsertL0(l0Record, supportsBgEmbed ? undefined : embedding);
|
||||
l0UpsertTotalMs += performance.now() - tUpsertStart;
|
||||
|
||||
if (upsertOk) {
|
||||
l0VectorsWritten++;
|
||||
if (supportsBgEmbed) {
|
||||
bgRecords.push({ recordId: l0Record.id, content: msg.content });
|
||||
}
|
||||
} else {
|
||||
logger?.warn(`${TAG} [L0-vec-index] upsertL0 returned false for message ${i}`);
|
||||
}
|
||||
} catch (err) {
|
||||
logger?.warn?.(`${TAG} [L0-vec-index] FAILED for message ${i} (non-blocking): ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
}
|
||||
|
||||
const modeLabel = supportsBgEmbed ? "metadata-only, embed=background" : `embed=${l0EmbedTotalMs.toFixed(0)}ms, upsert=${l0UpsertTotalMs.toFixed(0)}ms`;
|
||||
logger?.debug?.(`${TAG} [L0-vec-index] DONE: ${l0VectorsWritten}/${filteredMessages.length} records written (${modeLabel})`);
|
||||
|
||||
// Path A only: fire-and-forget background embedding for sqlite stores
|
||||
if (supportsBgEmbed && bgRecords.length > 0 && embeddingService) {
|
||||
const bgVectorStore = vectorStore;
|
||||
const bgEmbeddingService = embeddingService;
|
||||
const bgSnapshot = [...bgRecords];
|
||||
const bgLogger = logger;
|
||||
|
||||
// Do NOT await — runs in background after response is sent.
|
||||
//
|
||||
// Register the task in bgTaskRegistry (if provided) so TdaiCore.destroy()
|
||||
// can await it before closing vectorStore / embeddingService. The
|
||||
// ``.finally`` clean-up ensures the entry is removed on both success
|
||||
// and failure; without that the set would leak and eventually block
|
||||
// shutdown indefinitely.
|
||||
const bgPromise: Promise<void> = (async () => {
|
||||
const tBgStart = performance.now();
|
||||
try {
|
||||
const texts = bgSnapshot.map((r) => r.content);
|
||||
const embeddings = await bgEmbeddingService.embedBatch(texts);
|
||||
|
||||
let bgUpdated = 0;
|
||||
for (let i = 0; i < bgSnapshot.length; i++) {
|
||||
try {
|
||||
const ok = await bgVectorStore.updateL0Embedding!(bgSnapshot[i].recordId, embeddings[i]);
|
||||
if (ok) bgUpdated++;
|
||||
} catch (err) {
|
||||
bgLogger?.warn?.(
|
||||
`${TAG} [L0-vec-index-bg] Failed to update embedding for ${bgSnapshot[i].recordId}: ` +
|
||||
`${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
const bgMs = performance.now() - tBgStart;
|
||||
bgLogger?.debug?.(
|
||||
`${TAG} [L0-vec-index-bg] Background embedding complete: ${bgUpdated}/${bgSnapshot.length} vectors updated (${bgMs.toFixed(0)}ms)`,
|
||||
);
|
||||
} catch (err) {
|
||||
const bgMs = performance.now() - tBgStart;
|
||||
bgLogger?.warn?.(
|
||||
`${TAG} [L0-vec-index-bg] Background embedding failed (${bgMs.toFixed(0)}ms, non-fatal): ` +
|
||||
`${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
}
|
||||
})();
|
||||
|
||||
if (bgTaskRegistry) {
|
||||
bgTaskRegistry.add(bgPromise);
|
||||
void bgPromise.finally(() => {
|
||||
bgTaskRegistry.delete(bgPromise);
|
||||
});
|
||||
}
|
||||
}
|
||||
} else if (filteredMessages.length > 0) {
|
||||
logger?.warn(`${TAG} [L0-vec-index] SKIPPED: vectorStore not available`);
|
||||
}
|
||||
const tL0VecEnd = performance.now();
|
||||
|
||||
// ============================
|
||||
// Step 3: Notify scheduler of this conversation round
|
||||
// ============================
|
||||
const tNotifyStart = performance.now();
|
||||
// Pass empty array: L1 Runner reads from VectorStore DB (or L0 JSONL fallback), not from in-memory buffers.
|
||||
if (scheduler) {
|
||||
await scheduler.notifyConversation(sessionKey, []);
|
||||
logger?.debug?.(`${TAG} Scheduler notified of conversation round (sessionKey=${sessionKey})`);
|
||||
|
||||
const totalMs = performance.now() - tCaptureStart;
|
||||
const vecDetail = supportsBgEmbed
|
||||
? `metadata-only, embed=background, msgs=${filteredMessages.length}`
|
||||
: `embed=${l0EmbedTotalMs.toFixed(0)}ms, upsert=${l0UpsertTotalMs.toFixed(0)}ms, msgs=${filteredMessages.length}`;
|
||||
logger?.info(
|
||||
`${TAG} ⏱ Capture timing: total=${totalMs.toFixed(0)}ms, ` +
|
||||
`l0Record+checkpoint=${(tL0RecordEnd - tL0RecordStart).toFixed(0)}ms, ` +
|
||||
`l0VecIndex=${(tL0VecEnd - tL0VecStart).toFixed(0)}ms (${vecDetail}), ` +
|
||||
`notify=${(performance.now() - tNotifyStart).toFixed(0)}ms`,
|
||||
);
|
||||
|
||||
return {
|
||||
schedulerNotified: true,
|
||||
l0RecordedCount: filteredMessages.length,
|
||||
l0VectorsWritten,
|
||||
filteredMessages,
|
||||
};
|
||||
}
|
||||
|
||||
const totalMs = performance.now() - tCaptureStart;
|
||||
const vecDetail = supportsBgEmbed
|
||||
? `metadata-only, embed=background, msgs=${filteredMessages.length}`
|
||||
: `embed=${l0EmbedTotalMs.toFixed(0)}ms, upsert=${l0UpsertTotalMs.toFixed(0)}ms, msgs=${filteredMessages.length}`;
|
||||
logger?.info(
|
||||
`${TAG} ⏱ Capture timing: total=${totalMs.toFixed(0)}ms, ` +
|
||||
`l0Record+checkpoint=${(tL0RecordEnd - tL0RecordStart).toFixed(0)}ms, ` +
|
||||
`l0VecIndex=${(tL0VecEnd - tL0VecStart).toFixed(0)}ms (${vecDetail}), ` +
|
||||
`notify=${(performance.now() - tNotifyStart).toFixed(0)}ms`,
|
||||
);
|
||||
|
||||
logger?.debug?.(`${TAG} No scheduler provided, skipping notification`);
|
||||
return {
|
||||
schedulerNotified: false,
|
||||
l0RecordedCount: filteredMessages.length,
|
||||
l0VectorsWritten,
|
||||
filteredMessages,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,778 @@
|
||||
/**
|
||||
* auto-recall hook (v3): injects relevant memories + persona into agent context
|
||||
* before the agent starts processing.
|
||||
*
|
||||
* - Searches L1 memories using configurable strategy (keyword / embedding / hybrid)
|
||||
* - keyword: FTS5 BM25 (requires FTS5; returns empty if unavailable)
|
||||
* - embedding: VectorStore cosine similarity
|
||||
* - hybrid: keyword + embedding merged with RRF
|
||||
* - L3 persona injection
|
||||
* - L2 scene navigation (full injection, LLM decides relevance)
|
||||
*/
|
||||
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import type { MemoryTdaiConfig } from "../../config.js";
|
||||
import { readSceneIndex } from "../scene/scene-index.js";
|
||||
import { generateSceneNavigation, stripSceneNavigation } from "../scene/scene-navigation.js";
|
||||
import type { MemoryRecord } from "../record/l1-reader.js";
|
||||
import type { IMemoryStore, L1SearchResult, L1FtsResult } from "../store/types.js";
|
||||
import { buildFtsQuery } from "../store/sqlite.js";
|
||||
import type { EmbeddingService, EmbeddingCallOptions } from "../store/embedding.js";
|
||||
import { sanitizeText } from "../../utils/sanitize.js";
|
||||
|
||||
const TAG = "[memory-tdai] [recall]";
|
||||
|
||||
/**
|
||||
* Memory tools usage guide — injected at the end of memory context so the
|
||||
* main agent knows how to actively retrieve deeper information.
|
||||
*/
|
||||
const MEMORY_TOOLS_GUIDE = `<memory-tools-guide>
|
||||
## 记忆工具调用指南
|
||||
|
||||
当上方注入的记忆片段不足以回答用户问题时，可主动调用以下工具获取更多信息：
|
||||
|
||||
- **tdai_memory_search**：搜索结构化记忆（L1），适用于回忆用户偏好、历史事件节点、规则等关键信息。
|
||||
- **tdai_conversation_search**：搜索原始对话（L0），适用于查找具体消息原文、时间线、上下文细节；也可用于补充或校验 memory_search 的结果。
|
||||
- **read_file**（Scene Navigation 中的路径）：当已定位到相关情境，且需要该场景的完整画像、事件经过或阶段结论时使用。
|
||||
|
||||
### ⚠️ 调用次数限制
|
||||
每轮对话中，tdai_memory_search 和 tdai_conversation_search **合计最多调用 3 次**。
|
||||
- 首次搜索无结果时，可换关键词或换工具重试，但总调用次数不要超过 3 次。
|
||||
- 若 3 次搜索后仍无结果，说明该信息不在记忆中，请直接根据已有信息回复用户，不要继续搜索。
|
||||
</memory-tools-guide>`
|
||||
|
||||
interface Logger {
|
||||
debug?: (message: string) => void;
|
||||
info: (message: string) => void;
|
||||
warn: (message: string) => void;
|
||||
error: (message: string) => void;
|
||||
}
|
||||
|
||||
/** A single recalled L1 memory with its search score and type. */
|
||||
export interface RecalledMemory {
|
||||
content: string;
|
||||
score: number;
|
||||
type: string;
|
||||
}
|
||||
|
||||
export interface RecallResult {
|
||||
/** L1 relevant memories — prepended to user prompt text (dynamic, per-turn) */
|
||||
prependContext?: string;
|
||||
/** Stable recall context appended to system prompt (persona, scene nav, tools guide — cacheable) */
|
||||
appendSystemContext?: string;
|
||||
|
||||
// ── Metric payload (for pendingRecallCache in index.ts) ──
|
||||
/** L1 memories that were recalled (with scores), for metric reporting */
|
||||
recalledL1Memories?: RecalledMemory[];
|
||||
/** L3 Persona raw content loaded during recall (null if none) */
|
||||
recalledL3Persona?: string | null;
|
||||
/** Effective search strategy used */
|
||||
recallStrategy?: string;
|
||||
}
|
||||
|
||||
export async function performAutoRecall(params: {
|
||||
userText: string;
|
||||
actorId: string;
|
||||
sessionKey: string;
|
||||
cfg: MemoryTdaiConfig;
|
||||
pluginDataDir: string;
|
||||
logger?: Logger;
|
||||
vectorStore?: IMemoryStore;
|
||||
embeddingService?: EmbeddingService;
|
||||
}): Promise<RecallResult | undefined> {
|
||||
const { cfg, logger } = params;
|
||||
const timeoutMs = cfg.recall.timeoutMs ?? 5000;
|
||||
|
||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||
|
||||
return Promise.race([
|
||||
performAutoRecallInner(params).finally(() => {
|
||||
if (timer) clearTimeout(timer);
|
||||
}),
|
||||
new Promise<undefined>((resolve) => {
|
||||
timer = setTimeout(() => {
|
||||
logger?.warn?.(
|
||||
`${TAG} ⚠️ Recall timed out after ${timeoutMs}ms — skipping memory injection to avoid blocking the user`,
|
||||
);
|
||||
resolve(undefined);
|
||||
}, timeoutMs);
|
||||
}),
|
||||
]);
|
||||
}
|
||||
|
||||
async function performAutoRecallInner(params: {
|
||||
userText: string;
|
||||
actorId: string;
|
||||
sessionKey: string;
|
||||
cfg: MemoryTdaiConfig;
|
||||
pluginDataDir: string;
|
||||
logger?: Logger;
|
||||
vectorStore?: IMemoryStore;
|
||||
embeddingService?: EmbeddingService;
|
||||
}): Promise<RecallResult | undefined> {
|
||||
const { userText, cfg, pluginDataDir, logger, vectorStore, embeddingService } = params;
|
||||
const tRecallStart = performance.now();
|
||||
|
||||
// Search relevant memories (L1 layer) — skip only when userText is empty/undefined
|
||||
const tSearchStart = performance.now();
|
||||
let memoryLines: string[] = [];
|
||||
let effectiveStrategy = "skipped";
|
||||
let recalledL1Memories: RecalledMemory[] = [];
|
||||
let searchTiming: SearchTiming = { ftsMs: 0, embeddingMs: 0, ftsHits: 0, embeddingHits: 0 };
|
||||
if (!userText || userText.length === 0) {
|
||||
logger?.debug?.(`${TAG} User text empty/undefined, skipping memory search (persona/scene still injected)`);
|
||||
} else {
|
||||
effectiveStrategy = cfg.recall.strategy ?? "hybrid";
|
||||
const searchResult = await searchMemories(userText, pluginDataDir, cfg, logger, effectiveStrategy as "keyword" | "embedding" | "hybrid", vectorStore, embeddingService);
|
||||
memoryLines = searchResult.lines;
|
||||
searchTiming = searchResult.timing;
|
||||
|
||||
// Extract structured RecalledMemory from formatted lines for metric reporting
|
||||
recalledL1Memories = memoryLines.map((line) => {
|
||||
const match = line.match(/^-\s+\[([^\]]+)\]\s+(.+?)(?:\s*\(活动时间:.*\))?$/);
|
||||
if (match) {
|
||||
const tag = match[1];
|
||||
const content = match[2].trim();
|
||||
const typePart = tag.includes("|") ? tag.split("|")[0] : tag;
|
||||
return { content, score: 0, type: typePart };
|
||||
}
|
||||
return { content: line, score: 0, type: "unknown" };
|
||||
});
|
||||
}
|
||||
const tSearchEnd = performance.now();
|
||||
|
||||
// Read persona (L3 layer)
|
||||
const tPersonaStart = performance.now();
|
||||
let personaContent: string | undefined;
|
||||
try {
|
||||
const personaPath = path.join(pluginDataDir, "persona.md");
|
||||
const raw = await fs.readFile(personaPath, "utf-8");
|
||||
personaContent = stripSceneNavigation(raw).trim();
|
||||
if (!personaContent) personaContent = undefined;
|
||||
logger?.debug?.(`${TAG} Persona loaded: ${personaContent ? `${personaContent.length} chars` : "empty"}`);
|
||||
} catch {
|
||||
logger?.debug?.(`${TAG} No persona file found (expected for new users)`);
|
||||
}
|
||||
const tPersonaEnd = performance.now();
|
||||
|
||||
// Load full scene navigation (L2 layer)
|
||||
const tSceneStart = performance.now();
|
||||
let sceneNavigation: string | undefined;
|
||||
try {
|
||||
const sceneIndex = await readSceneIndex(pluginDataDir);
|
||||
if (sceneIndex.length > 0) {
|
||||
sceneNavigation = generateSceneNavigation(sceneIndex, pluginDataDir);
|
||||
logger?.debug?.(`${TAG} Scene navigation generated: ${sceneIndex.length} scenes`);
|
||||
}
|
||||
} catch {
|
||||
logger?.debug?.(`${TAG} No scene index found`);
|
||||
}
|
||||
const tSceneEnd = performance.now();
|
||||
|
||||
if (memoryLines.length === 0 && !personaContent && !sceneNavigation) {
|
||||
const totalMs = performance.now() - tRecallStart;
|
||||
logger?.info(
|
||||
`${TAG} ⏱ Recall timing: total=${totalMs.toFixed(0)}ms, ` +
|
||||
`search=${(tSearchEnd - tSearchStart).toFixed(0)}ms(strategy=${effectiveStrategy},hits=${memoryLines.length},` +
|
||||
`fts=${searchTiming.ftsMs.toFixed(0)}ms/${searchTiming.ftsHits}hits,` +
|
||||
`vec=${searchTiming.embeddingMs.toFixed(0)}ms/${searchTiming.embeddingHits}hits), ` +
|
||||
`persona=${(tPersonaEnd - tPersonaStart).toFixed(0)}ms, ` +
|
||||
`scene=${(tSceneEnd - tSceneStart).toFixed(0)}ms — no context to inject`,
|
||||
);
|
||||
logger?.debug?.(`${TAG} No memories/persona/scenes to inject`);
|
||||
return undefined;
|
||||
}
|
||||
|
||||
// Split recall context into stable and dynamic parts to optimize prompt caching.
|
||||
//
|
||||
// appendSystemContext (system prompt end — stable, cacheable):
|
||||
// persona, scene navigation, memory tools guide
|
||||
// These change infrequently; when content is identical across turns,
|
||||
// providers with prompt caching (Anthropic/OpenAI) can cache this region.
|
||||
//
|
||||
// prependContext (user prompt prefix — dynamic, per-turn):
|
||||
// L1 relevant memories — different every turn, moved out of system prompt
|
||||
// so it doesn't bust the system prompt cache.
|
||||
const stableParts: string[] = [];
|
||||
if (personaContent) {
|
||||
stableParts.push(`<user-persona>\n${personaContent}\n</user-persona>`);
|
||||
}
|
||||
if (sceneNavigation) {
|
||||
stableParts.push(`<scene-navigation>\n${sceneNavigation}\n</scene-navigation>`);
|
||||
}
|
||||
|
||||
// Dynamic part: L1 relevant memories (changes every turn) → prependContext (user prompt)
|
||||
let prependContext: string | undefined;
|
||||
if (memoryLines.length > 0) {
|
||||
prependContext =
|
||||
`<relevant-memories>\n以下是当前对话召回的相关记忆，不代表当前任务进程，仅作为参考：\n\n${memoryLines.join("\n")}\n</relevant-memories>`;
|
||||
}
|
||||
|
||||
// Append memory tools usage guide to the stable part so the agent knows
|
||||
// how to actively retrieve deeper context when the injected snippets
|
||||
// are not enough. This is static content and benefits from caching.
|
||||
if (stableParts.length > 0 || prependContext) {
|
||||
stableParts.push(MEMORY_TOOLS_GUIDE);
|
||||
}
|
||||
|
||||
const appendSystemContext = stableParts.length > 0 ? stableParts.join("\n\n") : undefined;
|
||||
|
||||
const totalMs = performance.now() - tRecallStart;
|
||||
logger?.info(
|
||||
`${TAG} ⏱ Recall timing: total=${totalMs.toFixed(0)}ms, ` +
|
||||
`search=${(tSearchEnd - tSearchStart).toFixed(0)}ms(strategy=${effectiveStrategy},hits=${memoryLines.length},` +
|
||||
`fts=${searchTiming.ftsMs.toFixed(0)}ms/${searchTiming.ftsHits}hits,` +
|
||||
`vec=${searchTiming.embeddingMs.toFixed(0)}ms/${searchTiming.embeddingHits}hits), ` +
|
||||
`persona=${(tPersonaEnd - tPersonaStart).toFixed(0)}ms(${personaContent ? `${personaContent.length}chars` : "none"}), ` +
|
||||
`scene=${(tSceneEnd - tSceneStart).toFixed(0)}ms(${sceneNavigation ? "loaded" : "none"})`,
|
||||
);
|
||||
|
||||
if (!appendSystemContext && !prependContext) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return {
|
||||
prependContext,
|
||||
appendSystemContext,
|
||||
recalledL1Memories,
|
||||
recalledL3Persona: personaContent ?? null,
|
||||
recallStrategy: effectiveStrategy,
|
||||
};
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Multi-strategy search dispatcher
|
||||
// ============================
|
||||
|
||||
interface ScoredRecord {
|
||||
record: MemoryRecord;
|
||||
score: number;
|
||||
}
|
||||
|
||||
/** Timing breakdown from memory search */
|
||||
interface SearchTiming {
|
||||
ftsMs: number;
|
||||
embeddingMs: number;
|
||||
ftsHits: number;
|
||||
embeddingHits: number;
|
||||
}
|
||||
|
||||
interface SearchResult {
|
||||
lines: string[];
|
||||
timing: SearchTiming;
|
||||
}
|
||||
|
||||
/**
|
||||
* Search memories and return both formatted lines and structured details.
|
||||
*
|
||||
* This is a thin wrapper around `searchMemories` that also captures
|
||||
* the recalled memory metadata for metric reporting (agent_turn event).
|
||||
* It parses the returned formatted lines to extract type/content info.
|
||||
*/
|
||||
async function searchMemoriesWithDetails(
|
||||
userText: string,
|
||||
pluginDataDir: string,
|
||||
cfg: MemoryTdaiConfig,
|
||||
logger: Logger | undefined,
|
||||
strategy: "keyword" | "embedding" | "hybrid",
|
||||
vectorStore?: IMemoryStore,
|
||||
embeddingService?: EmbeddingService,
|
||||
): Promise<{ lines: string[]; memories: RecalledMemory[]; timing: SearchTiming }> {
|
||||
const result = await searchMemories(userText, pluginDataDir, cfg, logger, strategy, vectorStore, embeddingService);
|
||||
|
||||
// Extract structured data from formatted memory lines.
|
||||
// Format: "- [type|scene] content (活动时间: ...)" or "- [type] content"
|
||||
const memories: RecalledMemory[] = result.lines.map((line) => {
|
||||
const match = line.match(/^-\s+\[([^\]]+)\]\s+(.+?)(?:\s*\(活动时间:.*\))?$/);
|
||||
if (match) {
|
||||
const tag = match[1];
|
||||
const content = match[2].trim();
|
||||
const typePart = tag.includes("|") ? tag.split("|")[0] : tag;
|
||||
return { content, score: 0, type: typePart };
|
||||
}
|
||||
return { content: line, score: 0, type: "unknown" };
|
||||
});
|
||||
|
||||
return { lines: result.lines, memories, timing: result.timing };
|
||||
}
|
||||
|
||||
/**
|
||||
* Search memories using the configured strategy.
|
||||
*
|
||||
* - "keyword": JSONL keyword-based (Jaccard similarity) — no embedding needed
|
||||
* - "embedding": VectorStore cosine similarity — requires vectorStore + embeddingService
|
||||
* - "hybrid": merge both keyword and embedding results with RRF (Reciprocal Rank Fusion)
|
||||
*
|
||||
* Falls back to keyword if embedding resources are unavailable.
|
||||
*/
|
||||
async function searchMemories(
|
||||
userText: string,
|
||||
pluginDataDir: string,
|
||||
cfg: MemoryTdaiConfig,
|
||||
logger: Logger | undefined,
|
||||
strategy: "keyword" | "embedding" | "hybrid",
|
||||
vectorStore?: IMemoryStore,
|
||||
embeddingService?: EmbeddingService,
|
||||
): Promise<SearchResult> {
|
||||
const emptyResult: SearchResult = { lines: [], timing: { ftsMs: 0, embeddingMs: 0, ftsHits: 0, embeddingHits: 0 } };
|
||||
// Strip gateway-injected inbound metadata (Sender, timestamps, media markers,
|
||||
// base64 image data, etc.) so FTS / embedding queries are based on pure user intent.
|
||||
const cleanText = sanitizeText(userText);
|
||||
|
||||
if (cleanText.length < 2) {
|
||||
logger?.debug?.(`${TAG} Query too short for memory search (raw=${userText.length}, clean=${cleanText.length})`);
|
||||
return emptyResult;
|
||||
}
|
||||
|
||||
if (cleanText.length !== userText.length) {
|
||||
logger?.debug?.(
|
||||
`${TAG} userText sanitized: ${userText.length} → ${cleanText.length} chars`,
|
||||
);
|
||||
}
|
||||
|
||||
const maxResults = cfg.recall.maxResults ?? 5;
|
||||
const threshold = cfg.recall.scoreThreshold ?? 0.3;
|
||||
|
||||
const embeddingAvailable = !!vectorStore && !!embeddingService;
|
||||
|
||||
logger?.debug?.(
|
||||
`${TAG} [searchMemories] strategy=${strategy}, embeddingAvailable=${embeddingAvailable}, ` +
|
||||
`vectorStore=${vectorStore ? "available" : "UNAVAILABLE"}, ` +
|
||||
`embeddingService=${embeddingService ? "available" : "UNAVAILABLE"}, ` +
|
||||
`maxResults=${maxResults}, threshold=${threshold}`,
|
||||
);
|
||||
|
||||
// Determine effective strategy (fall back to keyword if embedding not available)
|
||||
let effectiveStrategy = strategy;
|
||||
if ((strategy === "embedding" || strategy === "hybrid") && !embeddingAvailable) {
|
||||
logger?.warn?.(
|
||||
`${TAG} Strategy "${strategy}" requested but EmbeddingService not available, falling back to keyword`,
|
||||
);
|
||||
effectiveStrategy = "keyword";
|
||||
}
|
||||
|
||||
logger?.debug?.(`${TAG} Search strategy: ${effectiveStrategy} (configured: ${strategy})`);
|
||||
|
||||
// Resolve per-call embedding timeout for recall path.
|
||||
// Falls back to global embedding.timeoutMs when recallTimeoutMs is not configured.
|
||||
const recallEmbeddingTimeoutMs = cfg.embedding.recallTimeoutMs ?? cfg.embedding.timeoutMs;
|
||||
const embeddingCallOpts: EmbeddingCallOptions = { timeoutMs: recallEmbeddingTimeoutMs };
|
||||
|
||||
try {
|
||||
if (effectiveStrategy === "keyword") {
|
||||
const tFts = performance.now();
|
||||
const lines = await searchByKeyword(cleanText, pluginDataDir, maxResults, threshold, logger, vectorStore);
|
||||
return { lines, timing: { ftsMs: performance.now() - tFts, embeddingMs: 0, ftsHits: lines.length, embeddingHits: 0 } };
|
||||
}
|
||||
|
||||
if (effectiveStrategy === "embedding") {
|
||||
const tEmb = performance.now();
|
||||
const lines = await searchByEmbedding(cleanText, maxResults, threshold, vectorStore!, embeddingService!, logger, embeddingCallOpts);
|
||||
return { lines, timing: { ftsMs: 0, embeddingMs: performance.now() - tEmb, ftsHits: 0, embeddingHits: lines.length } };
|
||||
}
|
||||
|
||||
// Hybrid: run both keyword and embedding, merge with RRF
|
||||
return await searchHybrid(cleanText, pluginDataDir, maxResults, threshold, vectorStore!, embeddingService!, logger, embeddingCallOpts);
|
||||
} catch (err) {
|
||||
logger?.warn?.(`${TAG} Memory search failed (strategy=${effectiveStrategy}): ${err instanceof Error ? err.message : String(err)}`);
|
||||
return emptyResult;
|
||||
}
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Strategy: Keyword (FTS5 BM25, no in-memory fallback)
|
||||
// ============================
|
||||
|
||||
async function searchByKeyword(
|
||||
userText: string,
|
||||
_pluginDataDir: string,
|
||||
maxResults: number,
|
||||
threshold: number,
|
||||
logger?: Logger,
|
||||
vectorStore?: IMemoryStore,
|
||||
): Promise<string[]> {
|
||||
// Prefer FTS5 if available
|
||||
if (vectorStore?.isFtsAvailable()) {
|
||||
const ftsQuery = buildFtsQuery(userText);
|
||||
if (ftsQuery) {
|
||||
logger?.debug?.(`${TAG} [keyword-fts] Using FTS5 BM25 search: query="${ftsQuery}"`);
|
||||
const ftsResults = await vectorStore.searchL1Fts(ftsQuery, maxResults * 2);
|
||||
if (ftsResults.length > 0) {
|
||||
logger?.debug?.(
|
||||
`${TAG} [keyword-fts] FTS5 raw results (${ftsResults.length}): ` +
|
||||
ftsResults.map((r) => `id=${r.record_id} score=${r.score.toFixed(6)}`).join(", "),
|
||||
);
|
||||
const filtered = ftsResults
|
||||
.filter((r) => r.score >= threshold)
|
||||
.slice(0, maxResults);
|
||||
|
||||
if (filtered.length > 0) {
|
||||
logger?.debug?.(`${TAG} [keyword-fts] FTS5 found ${filtered.length} results (from ${ftsResults.length} raw, threshold=${threshold})`);
|
||||
return filtered.map((r) => formatMemoryLine(ftsResultToFormatable(r)));
|
||||
}
|
||||
|
||||
// BM25 absolute scores are unreliable when the document set is very
|
||||
// small (e.g. 1–3 records) because IDF approaches 0. In that case,
|
||||
// trust FTS5's MATCH + rank ordering and return the top results anyway.
|
||||
if (ftsResults.length <= maxResults) {
|
||||
logger?.debug?.(
|
||||
`${TAG} [keyword-fts] All ${ftsResults.length} results below threshold=${threshold} ` +
|
||||
`but document set is small — returning all matched results`,
|
||||
);
|
||||
return ftsResults.slice(0, maxResults).map((r) => formatMemoryLine(ftsResultToFormatable(r)));
|
||||
}
|
||||
logger?.debug?.(`${TAG} [keyword-fts] FTS5 returned 0 results above threshold (from ${ftsResults.length} raw)`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// FTS5 not available or returned no results — skip in-memory fallback to avoid O(N) full scan
|
||||
logger?.debug?.(`${TAG} [keyword] FTS5 unavailable or no results, skipping keyword search`);
|
||||
return [];
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Strategy: Embedding (VectorStore cosine)
|
||||
// ============================
|
||||
|
||||
async function searchByEmbedding(
|
||||
userText: string,
|
||||
maxResults: number,
|
||||
threshold: number,
|
||||
vectorStore: IMemoryStore,
|
||||
embeddingService: EmbeddingService,
|
||||
logger?: Logger,
|
||||
embeddingCallOpts?: EmbeddingCallOptions,
|
||||
): Promise<string[]> {
|
||||
logger?.debug?.(
|
||||
`${TAG} [embedding-search] START query="${userText.slice(0, 80)}...", maxResults=${maxResults}, threshold=${threshold}`,
|
||||
);
|
||||
const queryEmbedding = await embeddingService.embed(userText, embeddingCallOpts);
|
||||
logger?.debug?.(
|
||||
`${TAG} [embedding-search] Query embedding OK: dims=${queryEmbedding.length}, ` +
|
||||
`norm=${Math.sqrt(Array.from(queryEmbedding).reduce((s, v) => s + v * v, 0)).toFixed(4)}, ` +
|
||||
`searching top-${maxResults * 2}...`,
|
||||
);
|
||||
// Retrieve more candidates for subsequent filtering
|
||||
const vecResults: L1SearchResult[] = await vectorStore.searchL1Vector(queryEmbedding, maxResults * 2);
|
||||
|
||||
if (vecResults.length === 0) {
|
||||
logger?.debug?.(`${TAG} [embedding-search] Returned 0 results`);
|
||||
return [];
|
||||
}
|
||||
|
||||
logger?.debug?.(`${TAG} [embedding-search] Got ${vecResults.length} candidates, filtering by threshold=${threshold}`);
|
||||
for (const r of vecResults) {
|
||||
logger?.debug?.(
|
||||
`${TAG} [embedding-search] candidate id=${r.record_id}, score=${r.score.toFixed(4)}, ` +
|
||||
`type=${r.type}, content="${r.content.slice(0, 60)}..."`,
|
||||
);
|
||||
}
|
||||
|
||||
const filtered = vecResults
|
||||
.filter((r) => r.score >= threshold)
|
||||
.slice(0, maxResults);
|
||||
|
||||
if (filtered.length > 0) {
|
||||
logger?.debug?.(`${TAG} [embedding-search] Found ${filtered.length} relevant memories above threshold (from ${vecResults.length} candidates)`);
|
||||
return filtered.map((r) => formatMemoryLine(vectorResultToFormatable(r)));
|
||||
}
|
||||
|
||||
logger?.debug?.(`${TAG} [embedding-search] No results above threshold ${threshold}`);
|
||||
return [];
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Strategy: Hybrid (Keyword + Embedding + RRF)
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Hybrid search: run keyword (FTS5) and embedding in parallel, merge with
|
||||
* Reciprocal Rank Fusion (RRF) to combine rank lists.
|
||||
*
|
||||
* RRF score for a record at rank r = 1 / (k + r), where k=60 is a constant.
|
||||
* If a record appears in both lists, its RRF scores are summed.
|
||||
*
|
||||
* If FTS5 is unavailable, the keyword side returns empty and RRF uses
|
||||
* embedding results only.
|
||||
*/
|
||||
async function searchHybrid(
|
||||
userText: string,
|
||||
_pluginDataDir: string,
|
||||
maxResults: number,
|
||||
_threshold: number,
|
||||
vectorStore: IMemoryStore,
|
||||
embeddingService: EmbeddingService,
|
||||
logger?: Logger,
|
||||
embeddingCallOpts?: EmbeddingCallOptions,
|
||||
): Promise<SearchResult> {
|
||||
// Run keyword and embedding searches in parallel
|
||||
const candidateK = maxResults * 3; // retrieve more for merging
|
||||
|
||||
const [keywordResult, embeddingResult] = await Promise.all([
|
||||
// Keyword search: FTS5 only (no in-memory fallback)
|
||||
(async () => {
|
||||
const tStart = performance.now();
|
||||
try {
|
||||
// Try FTS5 first
|
||||
if (vectorStore.isFtsAvailable()) {
|
||||
const ftsQuery = buildFtsQuery(userText);
|
||||
if (ftsQuery) {
|
||||
const ftsResults = await vectorStore.searchL1Fts(ftsQuery, candidateK);
|
||||
if (ftsResults.length > 0) {
|
||||
logger?.debug?.(`${TAG} [hybrid-keyword-fts] FTS5 found ${ftsResults.length} candidates`);
|
||||
// Convert FtsSearchResult to ScoredRecord for RRF merge
|
||||
const records = ftsResults.map((r): ScoredRecord => ({
|
||||
record: {
|
||||
id: r.record_id,
|
||||
content: r.content,
|
||||
type: r.type as MemoryRecord["type"],
|
||||
priority: r.priority,
|
||||
scene_name: r.scene_name,
|
||||
source_message_ids: [],
|
||||
metadata: r.metadata_json ? (() => { try { return JSON.parse(r.metadata_json); } catch { return {}; } })() : {},
|
||||
timestamps: [r.timestamp_str].filter(Boolean),
|
||||
createdAt: "",
|
||||
updatedAt: "",
|
||||
sessionKey: r.session_key,
|
||||
sessionId: r.session_id,
|
||||
},
|
||||
score: r.score,
|
||||
}));
|
||||
return { records, ms: performance.now() - tStart };
|
||||
}
|
||||
}
|
||||
}
|
||||
// FTS5 not available or returned no results — skip in-memory fallback
|
||||
logger?.debug?.(`${TAG} [hybrid-keyword] FTS5 unavailable or no results, skipping keyword part`);
|
||||
return { records: [] as ScoredRecord[], ms: performance.now() - tStart };
|
||||
} catch (err) {
|
||||
logger?.warn?.(`${TAG} Hybrid: keyword part failed: ${err instanceof Error ? err.message : String(err)}`);
|
||||
return { records: [] as ScoredRecord[], ms: performance.now() - tStart };
|
||||
}
|
||||
})(),
|
||||
// Embedding search
|
||||
(async () => {
|
||||
const tStart = performance.now();
|
||||
try {
|
||||
logger?.debug?.(`${TAG} [hybrid-embedding] Generating query embedding...`);
|
||||
const queryEmbedding = await embeddingService.embed(userText, embeddingCallOpts);
|
||||
logger?.debug?.(
|
||||
`${TAG} [hybrid-embedding] Embedding OK, dims=${queryEmbedding.length}, searching top-${candidateK}...`,
|
||||
);
|
||||
const results = await vectorStore.searchL1Vector(queryEmbedding, candidateK, userText);
|
||||
logger?.debug?.(`${TAG} [hybrid-embedding] Got ${results.length} candidates`);
|
||||
return { results, ms: performance.now() - tStart };
|
||||
} catch (err) {
|
||||
logger?.warn?.(`${TAG} Hybrid: embedding part failed: ${err instanceof Error ? err.message : String(err)}`);
|
||||
return { results: [] as L1SearchResult[], ms: performance.now() - tStart };
|
||||
}
|
||||
})(),
|
||||
]);
|
||||
|
||||
const keywordResults = keywordResult.records;
|
||||
const embeddingResults = embeddingResult.results;
|
||||
const timing: SearchTiming = {
|
||||
ftsMs: keywordResult.ms,
|
||||
embeddingMs: embeddingResult.ms,
|
||||
ftsHits: keywordResults.length,
|
||||
embeddingHits: embeddingResults.length,
|
||||
};
|
||||
|
||||
if (keywordResults.length === 0 && embeddingResults.length === 0) {
|
||||
logger?.debug?.(`${TAG} Hybrid search: both strategies returned 0 results`);
|
||||
return { lines: [], timing };
|
||||
}
|
||||
|
||||
// RRF merge: k=60 is a standard constant from the RRF paper
|
||||
const RRF_K = 60;
|
||||
|
||||
// Map: record_id → { rrfScore, formatable }
|
||||
const mergedMap = new Map<string, { rrfScore: number; formatable: FormatableMemory }>();
|
||||
|
||||
// Process keyword results
|
||||
for (let rank = 0; rank < keywordResults.length; rank++) {
|
||||
const r = keywordResults[rank];
|
||||
const id = r.record.id;
|
||||
const rrfScore = 1 / (RRF_K + rank + 1);
|
||||
const existing = mergedMap.get(id);
|
||||
if (existing) {
|
||||
existing.rrfScore += rrfScore;
|
||||
} else {
|
||||
mergedMap.set(id, { rrfScore, formatable: recordToFormatable(r.record) });
|
||||
}
|
||||
}
|
||||
|
||||
// Process embedding results
|
||||
for (let rank = 0; rank < embeddingResults.length; rank++) {
|
||||
const r = embeddingResults[rank];
|
||||
const id = r.record_id;
|
||||
const rrfScore = 1 / (RRF_K + rank + 1);
|
||||
const existing = mergedMap.get(id);
|
||||
if (existing) {
|
||||
existing.rrfScore += rrfScore;
|
||||
} else {
|
||||
mergedMap.set(id, { rrfScore, formatable: vectorResultToFormatable(r) });
|
||||
}
|
||||
}
|
||||
|
||||
// Sort by combined RRF score and take top results
|
||||
const sorted = [...mergedMap.entries()]
|
||||
.sort((a, b) => b[1].rrfScore - a[1].rrfScore)
|
||||
.slice(0, maxResults);
|
||||
|
||||
if (sorted.length > 0) {
|
||||
logger?.debug?.(
|
||||
`${TAG} Hybrid search found ${sorted.length} results ` +
|
||||
`(keyword=${keywordResults.length}, embedding=${embeddingResults.length})`,
|
||||
);
|
||||
return { lines: sorted.map(([, { formatable }]) => formatMemoryLine(formatable)), timing };
|
||||
}
|
||||
|
||||
logger?.debug?.(`${TAG} Hybrid search: no results after merge`);
|
||||
return { lines: [], timing };
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Unified memory line formatter
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Format a single memory record into a rich natural-language line for prompt injection.
|
||||
*
|
||||
* Time semantics:
|
||||
* - timestamp (点时间): when the activity/event happened, e.g. "2025-03-01 mentioned something"
|
||||
* - activity_start_time / activity_end_time (段时间): activity time range, e.g. "trip from 2025-05-01 to 2025-05-10"
|
||||
* - All three time fields may be empty/undefined — handled gracefully.
|
||||
*
|
||||
* Output examples:
|
||||
* - [persona] 用户叫王小明，30岁，是一名软件工程师。
|
||||
* - [episodic|旅行计划] 用户计划五月去日本旅行。(活动时间: 2025-05-01 ~ 2025-05-10)
|
||||
* - [episodic] 用户今天加班到很晚。(活动时间: 2025-03-01)
|
||||
* - [instruction] 用户要求回答时使用中文，保持简洁。
|
||||
*/
|
||||
interface FormatableMemory {
|
||||
type: string;
|
||||
content: string;
|
||||
scene_name?: string;
|
||||
/** Activity time range start (段时间 start), may be empty */
|
||||
activity_start_time?: string;
|
||||
/** Activity time range end (段时间 end), may be empty */
|
||||
activity_end_time?: string;
|
||||
/** Activity point-in-time (点时间: when it happened), may be empty */
|
||||
timestamp?: string;
|
||||
}
|
||||
|
||||
function formatMemoryLine(m: FormatableMemory): string {
|
||||
// 1. Type tag + optional scene name
|
||||
const tag = m.scene_name ? `${m.type}|${m.scene_name}` : m.type;
|
||||
|
||||
// 2. Content (core)
|
||||
let line = `- [${tag}] ${m.content}`;
|
||||
|
||||
// 3. Time info — prefer activity_start/end range; fall back to timestamp as point-in-time
|
||||
const start = formatTimestamp(m.activity_start_time);
|
||||
const end = formatTimestamp(m.activity_end_time);
|
||||
const point = formatTimestamp(m.timestamp);
|
||||
|
||||
if (start && end) {
|
||||
// 段时间: both start and end
|
||||
line += ` (活动时间: ${start} ~ ${end})`;
|
||||
} else if (start) {
|
||||
// 段时间: only start
|
||||
line += ` (活动时间: ${start}起)`;
|
||||
} else if (end) {
|
||||
// 段时间: only end
|
||||
line += ` (活动时间: 至${end})`;
|
||||
} else if (point) {
|
||||
// 点时间: single timestamp
|
||||
line += ` (活动时间: ${point})`;
|
||||
}
|
||||
// If all three are empty → no time info appended (graceful)
|
||||
|
||||
return line;
|
||||
}
|
||||
|
||||
/**
|
||||
* Format an ISO 8601 timestamp to a concise date or datetime string.
|
||||
* - If the time part is 00:00:00 → show date only (e.g. "2025-03-01")
|
||||
* - Otherwise → show date + time (e.g. "2025-03-01 14:30")
|
||||
* - Returns undefined for empty/invalid inputs.
|
||||
*/
|
||||
function formatTimestamp(ts: string | undefined): string | undefined {
|
||||
if (!ts) return undefined;
|
||||
// Try to parse ISO format: "2025-03-01T14:30:00.000Z" or "2025-03-01"
|
||||
const match = ts.match(/^(\d{4}-\d{2}-\d{2})(?:T(\d{2}:\d{2})(?::\d{2})?)?/);
|
||||
if (!match) return undefined;
|
||||
const datePart = match[1];
|
||||
const timePart = match[2];
|
||||
if (!timePart || timePart === "00:00") {
|
||||
return datePart;
|
||||
}
|
||||
return `${datePart} ${timePart}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a FormatableMemory from a full MemoryRecord (keyword search path).
|
||||
* Handles empty metadata, empty timestamps array gracefully.
|
||||
*/
|
||||
function recordToFormatable(record: MemoryRecord): FormatableMemory {
|
||||
const meta = record.metadata as { activity_start_time?: string; activity_end_time?: string } | undefined;
|
||||
return {
|
||||
type: record.type,
|
||||
content: record.content,
|
||||
scene_name: record.scene_name || undefined,
|
||||
activity_start_time: meta?.activity_start_time || undefined,
|
||||
activity_end_time: meta?.activity_end_time || undefined,
|
||||
timestamp: (record.timestamps && record.timestamps.length > 0) ? record.timestamps[0] : undefined,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a FormatableMemory from a VectorSearchResult (embedding search path).
|
||||
* Handles empty/invalid metadata_json, empty timestamp_str gracefully.
|
||||
*/
|
||||
function vectorResultToFormatable(r: L1SearchResult): FormatableMemory {
|
||||
let activityStart: string | undefined;
|
||||
let activityEnd: string | undefined;
|
||||
if (r.metadata_json && r.metadata_json !== "{}") {
|
||||
try {
|
||||
const meta = typeof r.metadata_json === "string" ? JSON.parse(r.metadata_json) : r.metadata_json;
|
||||
activityStart = meta?.activity_start_time || undefined;
|
||||
activityEnd = meta?.activity_end_time || undefined;
|
||||
} catch { /* ignore parse errors — treat as no metadata */ }
|
||||
}
|
||||
return {
|
||||
type: r.type,
|
||||
content: r.content,
|
||||
scene_name: r.scene_name || undefined,
|
||||
activity_start_time: activityStart,
|
||||
activity_end_time: activityEnd,
|
||||
timestamp: r.timestamp_str || undefined,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a FormatableMemory from an FtsSearchResult (FTS5 keyword search path).
|
||||
* Handles empty/invalid metadata_json, empty timestamp_str gracefully.
|
||||
*/
|
||||
function ftsResultToFormatable(r: L1FtsResult): FormatableMemory {
|
||||
let activityStart: string | undefined;
|
||||
let activityEnd: string | undefined;
|
||||
if (r.metadata_json && r.metadata_json !== "{}") {
|
||||
try {
|
||||
const meta = typeof r.metadata_json === "string" ? JSON.parse(r.metadata_json) : r.metadata_json;
|
||||
activityStart = meta?.activity_start_time || undefined;
|
||||
activityEnd = meta?.activity_end_time || undefined;
|
||||
} catch { /* ignore parse errors — treat as no metadata */ }
|
||||
}
|
||||
return {
|
||||
type: r.type,
|
||||
content: r.content,
|
||||
scene_name: r.scene_name || undefined,
|
||||
activity_start_time: activityStart,
|
||||
activity_end_time: activityEnd,
|
||||
timestamp: r.timestamp_str || undefined,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
/**
|
||||
* TDAI Core — barrel re-export for core types and service facade.
|
||||
*
|
||||
* This module exports ONLY the host-neutral interfaces and the TdaiCore facade.
|
||||
* Host-specific adapters live in `../adapters/`.
|
||||
*/
|
||||
|
||||
// Types & interfaces
|
||||
export type {
|
||||
Logger,
|
||||
RuntimeContext,
|
||||
LLMRunParams,
|
||||
LLMRunner,
|
||||
LLMRunnerCreateOptions,
|
||||
LLMRunnerFactory,
|
||||
HostAdapter,
|
||||
CompletedTurn,
|
||||
RecallResult,
|
||||
CaptureResult,
|
||||
MemorySearchParams,
|
||||
ConversationSearchParams,
|
||||
} from "./types.js";
|
||||
|
||||
// TdaiCore service facade
|
||||
export { TdaiCore } from "./tdai-core.js";
|
||||
export type { TdaiCoreOptions } from "./tdai-core.js";
|
||||
@@ -0,0 +1,224 @@
|
||||
/**
|
||||
* PersonaGenerator: generates or updates user persona using the four-layer
|
||||
* deep scan model via CleanContextRunner.
|
||||
*/
|
||||
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { CleanContextRunner } from "../../utils/clean-context-runner.js";
|
||||
import { CheckpointManager } from "../../utils/checkpoint.js";
|
||||
import { readSceneIndex } from "../scene/scene-index.js";
|
||||
import { generateSceneNavigation, stripSceneNavigation } from "../scene/scene-navigation.js";
|
||||
import { buildPersonaPrompt } from "../prompts/persona-generation.js";
|
||||
import { BackupManager } from "../../utils/backup.js";
|
||||
import { escapeXmlTags } from "../../utils/sanitize.js";
|
||||
import { report } from "../report/reporter.js";
|
||||
import type { LLMRunner } from "../types.js";
|
||||
|
||||
const TAG = "[memory-tdai] [persona]";
|
||||
|
||||
interface Logger {
|
||||
debug?: (message: string) => void;
|
||||
info: (message: string) => void;
|
||||
warn: (message: string) => void;
|
||||
error: (message: string) => void;
|
||||
}
|
||||
|
||||
export class PersonaGenerator {
|
||||
private dataDir: string;
|
||||
private runner: LLMRunner;
|
||||
private logger: Logger | undefined;
|
||||
private backupCount: number;
|
||||
private instanceId: string | undefined;
|
||||
|
||||
constructor(opts: {
|
||||
dataDir: string;
|
||||
config: unknown;
|
||||
model?: string;
|
||||
backupCount?: number;
|
||||
logger?: Logger;
|
||||
/** Plugin instance ID for metric reporting (optional) */
|
||||
instanceId?: string;
|
||||
/**
|
||||
* Host-neutral LLM runner. When provided, used instead of creating
|
||||
* a CleanContextRunner (decouples from OpenClaw runtime).
|
||||
* Must be configured with `enableTools: true`.
|
||||
*/
|
||||
llmRunner?: LLMRunner;
|
||||
}) {
|
||||
this.dataDir = opts.dataDir;
|
||||
this.logger = opts.logger;
|
||||
this.backupCount = opts.backupCount ?? 3;
|
||||
this.instanceId = opts.instanceId;
|
||||
// Use injected LLMRunner if available, otherwise fall back to CleanContextRunner
|
||||
this.runner = opts.llmRunner ?? new CleanContextRunner({
|
||||
config: opts.config,
|
||||
modelRef: opts.model,
|
||||
enableTools: true,
|
||||
logger: opts.logger,
|
||||
});
|
||||
this.logger?.debug?.(`${TAG} Generator created: model=${opts.model ?? "(default)"}, dataDir=${opts.dataDir}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute local persona generation without advancing checkpoint.
|
||||
*/
|
||||
async generateLocalPersona(triggerReason?: string): Promise<boolean> {
|
||||
const startMs = Date.now();
|
||||
this.logger?.debug?.(`${TAG} Starting generation: reason="${triggerReason ?? "none"}"`);
|
||||
|
||||
const cpManager = new CheckpointManager(this.dataDir);
|
||||
const cp = await cpManager.read();
|
||||
this.logger?.debug?.(`${TAG} Checkpoint: total_processed=${cp.total_processed}, last_persona_at=${cp.last_persona_at}`);
|
||||
|
||||
const personaPath = path.join(this.dataDir, "persona.md");
|
||||
|
||||
// 1. Read existing persona (strip navigation)
|
||||
let existingPersona: string | undefined;
|
||||
try {
|
||||
const raw = await fs.readFile(personaPath, "utf-8");
|
||||
existingPersona = stripSceneNavigation(raw).trim() || undefined;
|
||||
this.logger?.debug?.(`${TAG} Existing persona: ${existingPersona ? `${existingPersona.length} chars` : "empty"}`);
|
||||
} catch {
|
||||
this.logger?.debug?.(`${TAG} No existing persona file`);
|
||||
}
|
||||
|
||||
// 2. Load scene index + identify changed scenes
|
||||
const index = await readSceneIndex(this.dataDir);
|
||||
const changedScenes = index.filter((e) => {
|
||||
if (!cp.last_persona_time) return true;
|
||||
const updatedMs = new Date(e.updated).getTime();
|
||||
const personaMs = new Date(cp.last_persona_time).getTime();
|
||||
// If either date is unparseable (NaN), treat as changed (conservative)
|
||||
if (Number.isNaN(updatedMs) || Number.isNaN(personaMs)) return true;
|
||||
return updatedMs > personaMs;
|
||||
});
|
||||
this.logger?.debug?.(`${TAG} Scene index: ${index.length} total, ${changedScenes.length} changed since last persona`);
|
||||
|
||||
// 3. Read changed scene contents (full raw content including META, matching Python reference)
|
||||
const blocksDir = path.join(this.dataDir, "scene_blocks");
|
||||
const changedSceneContents: string[] = [];
|
||||
for (const entry of changedScenes) {
|
||||
try {
|
||||
const raw = await fs.readFile(path.join(blocksDir, entry.filename), "utf-8");
|
||||
changedSceneContents.push(
|
||||
`### [${changedSceneContents.length + 1}] ${entry.filename}\n\n\`\`\`markdown\n${raw}\n\`\`\``,
|
||||
);
|
||||
} catch {
|
||||
this.logger?.warn(`${TAG} Could not read scene block: ${entry.filename}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (changedSceneContents.length === 0 && existingPersona) {
|
||||
this.logger?.debug?.(`${TAG} No scene changes and persona exists, skipping generation`);
|
||||
return false;
|
||||
}
|
||||
|
||||
// 4. Determine mode
|
||||
const mode = existingPersona ? "incremental" : "first";
|
||||
this.logger?.debug?.(`${TAG} Generation mode: ${mode}, ${changedSceneContents.length} scene blocks to process`);
|
||||
|
||||
// 5. Build changed scenes section with guidance (matching Python reference format)
|
||||
let changedScenesContent: string;
|
||||
if (changedSceneContents.length > 0) {
|
||||
changedScenesContent =
|
||||
`\n\n## 📄 变化场景完整内容\n\n` +
|
||||
`*自上次 Persona 更新后，以下 ${changedSceneContents.length} 个场景发生了变化。工程已为你预加载完整内容：*\n\n` +
|
||||
changedSceneContents.join("\n\n") +
|
||||
`\n\n---\n\n` +
|
||||
`⚠️ **重点分析变化场景**：上述场景是自上次更新后的**新增/修改内容**，请**重点分析**这些场景中的新信息。\n`;
|
||||
} else {
|
||||
changedScenesContent = `\n\n⚠️ **无变化场景**：所有场景均已在上次 Persona 更新中分析过，本次可直接读取所有场景进行全局审视。\n`;
|
||||
}
|
||||
|
||||
// 6. Build prompt
|
||||
const { systemPrompt, userPrompt } = buildPersonaPrompt({
|
||||
mode,
|
||||
currentTime: new Date().toISOString(),
|
||||
totalProcessed: cp.total_processed,
|
||||
sceneCount: index.length,
|
||||
changedSceneCount: changedScenes.length,
|
||||
changedScenesContent,
|
||||
existingPersona,
|
||||
triggerInfo: triggerReason,
|
||||
personaFilePath: personaPath,
|
||||
checkpointPath: path.join(this.dataDir, ".metadata", "recall_checkpoint.json"),
|
||||
});
|
||||
|
||||
// 7. Backup before LLM run (LLM writes persona.md via tools)
|
||||
const bm = new BackupManager(path.join(this.dataDir, ".backup"));
|
||||
await bm.backupFile(personaPath, "persona", `offset${cp.total_processed}`, this.backupCount);
|
||||
|
||||
// 8. Run LLM agent (sandboxed to dataDir, tools enabled — LLM writes persona.md directly)
|
||||
try {
|
||||
this.logger?.debug?.(`${TAG} Calling LLM for persona generation (timeout=180s, tools=enabled, workspaceDir=${this.dataDir})...`);
|
||||
await this.runner.run({
|
||||
systemPrompt,
|
||||
prompt: userPrompt,
|
||||
taskId: "persona-generation",
|
||||
timeoutMs: 180_000,
|
||||
// maxTokens omitted → core uses the resolved model's maxTokens from catalog
|
||||
workspaceDir: this.dataDir,
|
||||
});
|
||||
this.logger?.debug?.(`${TAG} LLM runner completed`);
|
||||
} catch (err) {
|
||||
const elapsedMs = Date.now() - startMs;
|
||||
this.logger?.error(`${TAG} Persona generation failed after ${elapsedMs}ms: ${err instanceof Error ? err.stack ?? err.message : String(err)}`);
|
||||
return false;
|
||||
}
|
||||
|
||||
// 9. Read LLM-written persona.md and apply post-processing
|
||||
let personaText: string;
|
||||
try {
|
||||
personaText = await fs.readFile(personaPath, "utf-8");
|
||||
} catch {
|
||||
// LLM failed to write persona.md — treat as failure
|
||||
this.logger?.error(`${TAG} LLM did not write persona.md — file not found after runner completed`);
|
||||
return false;
|
||||
}
|
||||
|
||||
// 10. Strip any navigation the LLM might have added + sanitize for safe injection
|
||||
personaText = escapeXmlTags(stripSceneNavigation(personaText).trim());
|
||||
|
||||
if (!personaText) {
|
||||
this.logger?.error(`${TAG} LLM wrote empty persona.md — skipping`);
|
||||
return false;
|
||||
}
|
||||
|
||||
// 11. Append fresh scene navigation and write final content
|
||||
const nav = generateSceneNavigation(index);
|
||||
const finalContent = nav ? `${personaText}\n\n${nav}\n` : personaText;
|
||||
await fs.writeFile(personaPath, finalContent, "utf-8");
|
||||
|
||||
const elapsedMs = Date.now() - startMs;
|
||||
this.logger?.info(`${TAG} Persona written (${finalContent.length} chars) in ${elapsedMs}ms`);
|
||||
|
||||
// ── l3_persona_generation metric ──
|
||||
if (this.instanceId && this.logger) {
|
||||
report("l3_persona_generation", {
|
||||
triggerReason: triggerReason ?? "unknown",
|
||||
mode: existingPersona ? "incremental" : "initial",
|
||||
newPersonaContent: personaText,
|
||||
newPersonaLength: personaText.length,
|
||||
totalDurationMs: elapsedMs,
|
||||
success: true,
|
||||
error: null,
|
||||
});
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Backward-compatible wrapper: local generation + checkpoint advance.
|
||||
*/
|
||||
async generate(triggerReason?: string): Promise<boolean> {
|
||||
const updated = await this.generateLocalPersona(triggerReason);
|
||||
if (!updated) return false;
|
||||
|
||||
const cpManager = new CheckpointManager(this.dataDir);
|
||||
const cp = await cpManager.read();
|
||||
await cpManager.markPersonaGenerated(cp.total_processed);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
/**
|
||||
* PersonaTrigger: determines whether to trigger persona generation.
|
||||
* Implements the 5 trigger conditions from the legacy system.
|
||||
*/
|
||||
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { CheckpointManager } from "../../utils/checkpoint.js";
|
||||
import { stripSceneNavigation } from "../scene/scene-navigation.js";
|
||||
|
||||
const TAG = "[memory-tdai] [trigger]";
|
||||
|
||||
interface TriggerLogger {
|
||||
debug?: (message: string) => void;
|
||||
info: (message: string) => void;
|
||||
warn: (message: string) => void;
|
||||
error: (message: string) => void;
|
||||
}
|
||||
|
||||
export interface TriggerResult {
|
||||
should: boolean;
|
||||
reason: string;
|
||||
}
|
||||
|
||||
export class PersonaTrigger {
|
||||
private dataDir: string;
|
||||
private interval: number;
|
||||
private logger: TriggerLogger | undefined;
|
||||
|
||||
constructor(opts: { dataDir: string; interval: number; logger?: TriggerLogger }) {
|
||||
this.dataDir = opts.dataDir;
|
||||
this.interval = opts.interval;
|
||||
this.logger = opts.logger;
|
||||
}
|
||||
|
||||
async shouldGenerate(): Promise<TriggerResult> {
|
||||
const cpManager = new CheckpointManager(this.dataDir);
|
||||
const cp = await cpManager.read();
|
||||
this.logger?.debug?.(`${TAG} Evaluating: total_processed=${cp.total_processed}, last_persona_at=${cp.last_persona_at}, memories_since=${cp.memories_since_last_persona}, scenes=${cp.scenes_processed}`);
|
||||
|
||||
// Priority 1: Agent explicitly requested persona update
|
||||
if (cp.request_persona_update) {
|
||||
const result: TriggerResult = {
|
||||
should: true,
|
||||
reason: `主动请求: ${cp.persona_update_reason || "Agent 请求更新"}`,
|
||||
};
|
||||
this.logger?.debug?.(`${TAG} Trigger P1 (explicit request): ${result.reason}`);
|
||||
return result;
|
||||
}
|
||||
|
||||
// Priority 2: Cold start — first extraction done, no persona yet, has scene files
|
||||
if (
|
||||
cp.scenes_processed > 0 &&
|
||||
cp.last_persona_at === 0 &&
|
||||
(await this.hasSceneFiles())
|
||||
) {
|
||||
const result: TriggerResult = { should: true, reason: "首次冷启动：首次提取完成且有场景文件" };
|
||||
this.logger?.debug?.(`${TAG} Trigger P2 (cold start): scenes_processed=${cp.scenes_processed}, total_processed=${cp.total_processed}`);
|
||||
return result;
|
||||
}
|
||||
|
||||
// Priority 2.5: Recovery — persona was generated before but persona.md body
|
||||
// is now empty (corrupted/missing). Regenerate to restore.
|
||||
if (
|
||||
cp.last_persona_at > 0 &&
|
||||
(await this.hasSceneFiles()) &&
|
||||
!(await this.hasPersonaBody())
|
||||
) {
|
||||
const result: TriggerResult = { should: true, reason: "恢复：persona.md 正文丢失或为空，需要重新生成" };
|
||||
this.logger?.debug?.(`${TAG} Trigger P2.5 (recovery): last_persona_at=${cp.last_persona_at}, persona body missing`);
|
||||
return result;
|
||||
}
|
||||
|
||||
// Priority 3: First scene block extraction
|
||||
if (cp.scenes_processed === 1 && cp.memories_since_last_persona > 0) {
|
||||
const result: TriggerResult = { should: true, reason: "首次 Scene Block 提取完成" };
|
||||
this.logger?.debug?.(`${TAG} Trigger P3 (first scene): scenes_processed=${cp.scenes_processed}`);
|
||||
return result;
|
||||
}
|
||||
|
||||
// Priority 4: Reached threshold
|
||||
if (cp.memories_since_last_persona >= this.interval) {
|
||||
const result: TriggerResult = {
|
||||
should: true,
|
||||
reason: `达到阈值: ${cp.memories_since_last_persona} >= ${this.interval}`,
|
||||
};
|
||||
this.logger?.debug?.(`${TAG} Trigger P4 (threshold): ${result.reason}`);
|
||||
return result;
|
||||
}
|
||||
|
||||
this.logger?.debug?.(`${TAG} No trigger conditions met`);
|
||||
return { should: false, reason: "" };
|
||||
}
|
||||
|
||||
private async hasSceneFiles(): Promise<boolean> {
|
||||
const blocksDir = path.join(this.dataDir, "scene_blocks");
|
||||
try {
|
||||
const files = await fs.readdir(blocksDir);
|
||||
const hasFiles = files.some((f) => f.endsWith(".md"));
|
||||
return hasFiles;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check whether persona.md has a non-empty body (excluding scene navigation).
|
||||
* Returns false if the file doesn't exist, is empty, or only contains
|
||||
* scene navigation (no actual persona content).
|
||||
*/
|
||||
private async hasPersonaBody(): Promise<boolean> {
|
||||
const personaPath = path.join(this.dataDir, "persona.md");
|
||||
try {
|
||||
const raw = await fs.readFile(personaPath, "utf-8");
|
||||
const body = stripSceneNavigation(raw).trim();
|
||||
return body.length > 0;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,239 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import type { IMemoryStore, ProfileRecord, ProfileSyncRecord } from "../store/types.js";
|
||||
import { readSceneIndex, syncSceneIndex } from "../scene/scene-index.js";
|
||||
import { generateSceneNavigation, stripSceneNavigation } from "../scene/scene-navigation.js";
|
||||
|
||||
const PROFILE_SCOPE = "global";
|
||||
|
||||
/** Check if an error is a rename race condition (another concurrent pull won). */
|
||||
function isRenameRaceError(err: unknown): boolean {
|
||||
const code = (err as NodeJS.ErrnoException)?.code;
|
||||
return code === "ENOTEMPTY" || code === "EEXIST";
|
||||
}
|
||||
|
||||
interface Logger {
|
||||
debug?: (message: string) => void;
|
||||
info: (message: string) => void;
|
||||
warn: (message: string) => void;
|
||||
error: (message: string) => void;
|
||||
}
|
||||
|
||||
export interface ProfileBaseline {
|
||||
version: number;
|
||||
contentMd5: string;
|
||||
createdAtMs: number;
|
||||
}
|
||||
|
||||
export function buildProfileStableId(scope: string, type: "l2" | "l3", filename: string): string {
|
||||
const hash = createHash("sha256")
|
||||
.update(`${scope}\u0000${type}\u0000${filename}`)
|
||||
.digest("hex");
|
||||
return `profile:v1:${hash}`;
|
||||
}
|
||||
|
||||
function md5(text: string): string {
|
||||
return createHash("md5").update(text).digest("hex");
|
||||
}
|
||||
|
||||
async function statTimes(filePath: string): Promise<{ createdAtMs: number; updatedAtMs: number }> {
|
||||
try {
|
||||
const stat = await fs.stat(filePath);
|
||||
return {
|
||||
createdAtMs: Math.floor(stat.birthtimeMs || stat.ctimeMs || Date.now()),
|
||||
updatedAtMs: Math.floor(stat.mtimeMs || Date.now()),
|
||||
};
|
||||
} catch {
|
||||
const now = Date.now();
|
||||
return { createdAtMs: now, updatedAtMs: now };
|
||||
}
|
||||
}
|
||||
|
||||
async function refreshPersonaNavigation(dataDir: string): Promise<void> {
|
||||
const personaPath = path.join(dataDir, "persona.md");
|
||||
let body: string;
|
||||
try {
|
||||
body = stripSceneNavigation(await fs.readFile(personaPath, "utf-8")).trim();
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!body) return;
|
||||
|
||||
const index = await readSceneIndex(dataDir);
|
||||
const nav = generateSceneNavigation(index);
|
||||
const finalContent = nav ? `${body}\n\n${nav}\n` : `${body}\n`;
|
||||
await fs.writeFile(personaPath, finalContent, "utf-8");
|
||||
}
|
||||
|
||||
export async function listLocalProfiles(dataDir: string): Promise<ProfileRecord[]> {
|
||||
const profiles: ProfileRecord[] = [];
|
||||
const blocksDir = path.join(dataDir, "scene_blocks");
|
||||
|
||||
try {
|
||||
const files = (await fs.readdir(blocksDir)).filter((file) => file.endsWith(".md")).sort();
|
||||
for (const filename of files) {
|
||||
const filePath = path.join(blocksDir, filename);
|
||||
const content = await fs.readFile(filePath, "utf-8");
|
||||
const { createdAtMs, updatedAtMs } = await statTimes(filePath);
|
||||
profiles.push({
|
||||
id: buildProfileStableId(PROFILE_SCOPE, "l2", filename),
|
||||
type: "l2",
|
||||
filename,
|
||||
content,
|
||||
contentMd5: md5(content),
|
||||
version: 0,
|
||||
createdAtMs,
|
||||
updatedAtMs,
|
||||
});
|
||||
}
|
||||
} catch {
|
||||
// ignore missing scene_blocks directory
|
||||
}
|
||||
|
||||
const personaPath = path.join(dataDir, "persona.md");
|
||||
try {
|
||||
const rawPersona = await fs.readFile(personaPath, "utf-8");
|
||||
const body = stripSceneNavigation(rawPersona).trim();
|
||||
if (body) {
|
||||
const { createdAtMs, updatedAtMs } = await statTimes(personaPath);
|
||||
profiles.push({
|
||||
id: buildProfileStableId(PROFILE_SCOPE, "l3", "persona.md"),
|
||||
type: "l3",
|
||||
filename: "persona.md",
|
||||
content: body,
|
||||
contentMd5: md5(body),
|
||||
version: 0,
|
||||
createdAtMs,
|
||||
updatedAtMs,
|
||||
});
|
||||
}
|
||||
} catch {
|
||||
// ignore missing persona file
|
||||
}
|
||||
|
||||
return profiles;
|
||||
}
|
||||
|
||||
export async function pullProfilesToLocal(
|
||||
dataDir: string,
|
||||
store: IMemoryStore,
|
||||
logger: Logger,
|
||||
): Promise<Map<string, ProfileBaseline>> {
|
||||
if (!store.pullProfiles) return new Map();
|
||||
|
||||
const records = await store.pullProfiles();
|
||||
const baseline = new Map<string, ProfileBaseline>();
|
||||
const tempDir = await fs.mkdtemp(path.join(dataDir, ".profiles-pull-"));
|
||||
const tempBlocksDir = path.join(tempDir, "scene_blocks");
|
||||
await fs.mkdir(tempBlocksDir, { recursive: true });
|
||||
|
||||
try {
|
||||
for (const record of records) {
|
||||
baseline.set(record.id, {
|
||||
version: record.version,
|
||||
contentMd5: record.contentMd5,
|
||||
createdAtMs: record.createdAtMs,
|
||||
});
|
||||
|
||||
if (record.type === "l2") {
|
||||
const target = path.join(tempBlocksDir, record.filename);
|
||||
await fs.writeFile(target, record.content, "utf-8");
|
||||
if (md5(record.content) !== record.contentMd5) {
|
||||
await fs.rm(target, { force: true });
|
||||
logger.warn(`[memory-tdai][profile-sync] MD5 mismatch for ${record.filename}`);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (record.type === "l3") {
|
||||
const body = stripSceneNavigation(record.content).trim();
|
||||
await fs.writeFile(path.join(tempDir, "persona.md"), body, "utf-8");
|
||||
if (md5(body) !== record.contentMd5) {
|
||||
await fs.rm(path.join(tempDir, "persona.md"), { force: true });
|
||||
logger.warn(`[memory-tdai][profile-sync] MD5 mismatch for ${record.filename}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const localBlocksDir = path.join(dataDir, "scene_blocks");
|
||||
await fs.rm(localBlocksDir, { recursive: true, force: true });
|
||||
await fs.mkdir(path.dirname(localBlocksDir), { recursive: true });
|
||||
try {
|
||||
await fs.rename(tempBlocksDir, localBlocksDir);
|
||||
} catch (err) {
|
||||
if (isRenameRaceError(err)) {
|
||||
// Another concurrent pull already wrote scene_blocks — ours is redundant.
|
||||
// Both pulls fetched the same remote snapshot, so the other result is equivalent.
|
||||
logger.debug?.(`[memory-tdai][profile-sync] scene_blocks rename lost race (${(err as NodeJS.ErrnoException).code}), using existing`);
|
||||
return baseline;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
|
||||
const tempPersonaPath = path.join(tempDir, "persona.md");
|
||||
const localPersonaPath = path.join(dataDir, "persona.md");
|
||||
try {
|
||||
await fs.access(tempPersonaPath);
|
||||
await fs.rm(localPersonaPath, { force: true });
|
||||
try {
|
||||
await fs.rename(tempPersonaPath, localPersonaPath);
|
||||
} catch (err) {
|
||||
if (!isRenameRaceError(err)) throw err;
|
||||
logger.debug?.(`[memory-tdai][profile-sync] persona.md rename lost race, using existing`);
|
||||
}
|
||||
} catch (err) {
|
||||
// No temp persona file → remove local persona (remote has none)
|
||||
if ((err as NodeJS.ErrnoException).code === "ENOENT") {
|
||||
await fs.rm(localPersonaPath, { force: true });
|
||||
} else if (!isRenameRaceError(err)) {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
await syncSceneIndex(dataDir);
|
||||
await refreshPersonaNavigation(dataDir);
|
||||
logger.debug?.(`[memory-tdai][profile-sync] Pulled ${records.length} profile(s) to local cache`);
|
||||
return baseline;
|
||||
} finally {
|
||||
await fs.rm(tempDir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
export async function syncLocalProfilesToStore(
|
||||
dataDir: string,
|
||||
store: IMemoryStore,
|
||||
baselineMap: Map<string, ProfileBaseline>,
|
||||
logger: Logger,
|
||||
): Promise<void> {
|
||||
const localProfiles = await listLocalProfiles(dataDir);
|
||||
const localIds = new Set(localProfiles.map((profile) => profile.id));
|
||||
|
||||
const syncRecords: ProfileSyncRecord[] = localProfiles
|
||||
.filter((profile) => baselineMap.get(profile.id)?.contentMd5 !== profile.contentMd5 || !baselineMap.has(profile.id))
|
||||
.map((profile) => ({
|
||||
...profile,
|
||||
baselineVersion: baselineMap.get(profile.id)?.version,
|
||||
}));
|
||||
|
||||
if (syncRecords.length > 0 && store.syncProfiles) {
|
||||
await store.syncProfiles(syncRecords);
|
||||
logger.info(`[memory-tdai][profile-sync] Synced ${syncRecords.length} changed profile(s)`);
|
||||
}
|
||||
|
||||
const deletedIds = [...baselineMap.keys()].filter((id) => !localIds.has(id));
|
||||
if (deletedIds.length > 0 && store.deleteProfiles) {
|
||||
await store.deleteProfiles(deletedIds);
|
||||
logger.info(`[memory-tdai][profile-sync] Deleted ${deletedIds.length} stale profile(s)`);
|
||||
}
|
||||
}
|
||||
|
||||
export async function ensureL2L3Local(
|
||||
dataDir: string,
|
||||
store: IMemoryStore,
|
||||
logger: Logger,
|
||||
): Promise<Map<string, ProfileBaseline>> {
|
||||
if (!store.pullProfiles) return new Map();
|
||||
return pullProfilesToLocal(dataDir, store, logger);
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
/**
|
||||
* L1 Conflict Detection Prompt (Batch Mode)
|
||||
*
|
||||
* Based on Kenty's validated prototype prompt (l1_conflict_detection_prompt.md).
|
||||
* Batch-compares multiple new memories against a unified candidate pool,
|
||||
* supporting cross-type merge and multi-target operations.
|
||||
*/
|
||||
|
||||
import type { MemoryRecord, ExtractedMemory } from "../record/l1-writer.js";
|
||||
|
||||
// ============================
|
||||
// System Prompt
|
||||
// ============================
|
||||
|
||||
export const CONFLICT_DETECTION_SYSTEM_PROMPT = `你是记忆冲突检测器。批量比较多条【新记忆】与【统一候选记忆池】中的已有记忆，逐条决定如何处理。
|
||||
|
||||
## 核心规则
|
||||
|
||||
- **跨 type 合并**：不同 type（persona / episodic / instruction）的记忆如果语义上描述同一事实/事件，**可以合并**。
|
||||
- **多对多合并**：一条新记忆可以同时替换/合并候选池中的**多条**已有记忆（通过 target_ids 数组指定）。
|
||||
- 合并后你必须判断新记忆的最佳 type（merged_type）。
|
||||
|
||||
## 判断逻辑
|
||||
|
||||
1. **分辨记忆性质**：
|
||||
- **状态类**（persona/instruction）：偏好、特质、长期设定、相对稳定的事实、行为规则
|
||||
- **事件类**（episodic）：一次性经历、带时间点的客观记录，建议合并同一件事的前因后果
|
||||
|
||||
2. **判断是否同一事实/事件**：主体相同、主题一致、时间接近、scene_name 相似
|
||||
|
||||
3. **选择动作**：
|
||||
- "store"：视为新信息，新增当前记忆。
|
||||
- "skip"：已有记忆更好，新记忆无增量或更模糊，忽略当前记忆。
|
||||
- "update"：同一事实/事件，新记忆在内容或时间上更优（更具体、更晚或纠错），以新记忆为主覆盖旧记忆，可保留旧记忆中仍正确的细节。
|
||||
- "merge"：同一事实或同一演化过程，多条记忆信息互补且不矛盾，合并成一条更完整记忆，信息尽量不冗余。
|
||||
|
||||
4. **策略倾向**：
|
||||
- 状态类：多条描述同一偏好/特质 → 倾向 merge；无增量 → skip；明确更新 → update
|
||||
- 事件类：同一事件的前因后果、不同阶段 → 倾向 merge 为一条完整叙述；完全相同 → skip
|
||||
- 跨类型示例：一条 episodic "用户在 2018 年开始做播客" + 一条 persona "用户有播客制作经验" → 可 merge 为一条 persona 或 episodic（取决于信息侧重）
|
||||
|
||||
5. **timestamp 处理**：
|
||||
- merge / update 时，merged_timestamps 应包含**所有相关记忆的时间戳并集**（去重排序）
|
||||
- 这样可以保留事件发生的完整时间线
|
||||
|
||||
## 输出格式
|
||||
|
||||
严格输出 JSON 数组，每个元素对应一条新记忆的决策。不输出任何其他内容：
|
||||
|
||||
[
|
||||
{
|
||||
"record_id": "新记忆的 record_id",
|
||||
"action": "store|update|skip|merge",
|
||||
"target_ids": ["要删除的候选记忆 record_id 1", "record_id 2"],
|
||||
"merged_content": "合并/更新后的记忆内容（merge/update 时必填）",
|
||||
"merged_type": "合并后的最佳 type：persona|episodic|instruction（merge/update 时必填）",
|
||||
"merged_priority": 85,
|
||||
"merged_timestamps": ["合并后的时间戳数组，包含所有新旧记忆时间戳的并集（merge/update 时必填）"]
|
||||
}
|
||||
]
|
||||
|
||||
字段说明：
|
||||
- target_ids：要删除替换的旧记忆 ID **数组**（可以 1 条或多条）。store/skip 时省略或为空。
|
||||
- merged_content：merge/update 时的最终记忆文本。store/skip 时省略。
|
||||
- merged_type：merge/update 后记忆应归属的 type。根据合并后内容本质判断。
|
||||
- merged_priority：merge/update 后的新优先级（0-100 整数，merge/update 时必填）。合并后信息更完整、更确定，通常应**酌情提升** priority（例如两条 priority 70 的记忆合并后可提升到 80）。参考标准：80-100（核心特质/重要事件），60-79（一般偏好/普通活动），<60（次要信息）。
|
||||
- merged_timestamps：合并后的时间戳数组。收集新记忆 + 所有被合并旧记忆的时间戳，去重排序。`;
|
||||
|
||||
// ============================
|
||||
// Prompt Builder
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Candidate search result for a single new memory.
|
||||
*/
|
||||
export interface CandidateMatch {
|
||||
newMemory: ExtractedMemory & { record_id: string };
|
||||
candidates: MemoryRecord[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Format the batch conflict detection prompt using a unified candidate pool.
|
||||
*
|
||||
* Format (aligned with prototype):
|
||||
* 1. Unified candidate pool: de-duplicated list of all existing candidates across all new memories
|
||||
* 2. Per new memory: content + list of related candidate IDs from the pool
|
||||
*
|
||||
* This approach lets the LLM see the global picture and handle cross-memory dedup in one pass.
|
||||
*
|
||||
* @param matches - Array of new memories with their candidate matches
|
||||
*/
|
||||
export function formatBatchConflictPrompt(matches: CandidateMatch[]): string {
|
||||
// Step 1: Build unified candidate pool (de-duplicate across all new memories)
|
||||
const unifiedPool = new Map<string, MemoryRecord>();
|
||||
const perMemoryCandidateIds = new Map<string, string[]>();
|
||||
|
||||
for (const m of matches) {
|
||||
const candidateIds: string[] = [];
|
||||
for (const c of m.candidates) {
|
||||
if (!unifiedPool.has(c.id)) {
|
||||
unifiedPool.set(c.id, c);
|
||||
}
|
||||
candidateIds.push(c.id);
|
||||
}
|
||||
perMemoryCandidateIds.set(m.newMemory.record_id, candidateIds);
|
||||
}
|
||||
|
||||
// Step 2: Format unified pool as JSON
|
||||
const poolList = Array.from(unifiedPool.values()).map((c) => ({
|
||||
record_id: c.id,
|
||||
content: c.content,
|
||||
type: c.type,
|
||||
priority: c.priority,
|
||||
scene_name: c.scene_name,
|
||||
timestamps: c.timestamps,
|
||||
}));
|
||||
|
||||
let poolSection: string;
|
||||
if (poolList.length === 0) {
|
||||
poolSection = "## 统一候选记忆池\n\n（空，没有已有记忆，所有新记忆直接 store）";
|
||||
} else {
|
||||
const poolStr = JSON.stringify(poolList, null, 2);
|
||||
poolSection = `## 统一候选记忆池（共 ${poolList.length} 条已有记忆）\n\n${poolStr}`;
|
||||
}
|
||||
|
||||
// Step 3: Format each new memory with its related candidate IDs
|
||||
const memoryParts = matches.map((m, idx) => {
|
||||
const relatedIds = perMemoryCandidateIds.get(m.newMemory.record_id) ?? [];
|
||||
const relatedNote =
|
||||
relatedIds.length > 0
|
||||
? JSON.stringify(relatedIds)
|
||||
: "[]（无相似候选，直接 store）";
|
||||
|
||||
const memStr = JSON.stringify(
|
||||
{
|
||||
record_id: m.newMemory.record_id,
|
||||
content: m.newMemory.content,
|
||||
type: m.newMemory.type,
|
||||
priority: m.newMemory.priority,
|
||||
scene_name: m.newMemory.scene_name,
|
||||
},
|
||||
null,
|
||||
2,
|
||||
);
|
||||
|
||||
return `### 第 ${idx + 1} 条新记忆 (record_id: ${m.newMemory.record_id})\n${memStr}\n\n【关联候选 ID】${relatedNote}`;
|
||||
});
|
||||
|
||||
const newMemoriesText = memoryParts.join(
|
||||
"\n\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n\n",
|
||||
);
|
||||
|
||||
// Step 4: Assemble final prompt
|
||||
return `${poolSection}
|
||||
|
||||
${"═".repeat(50)}
|
||||
|
||||
## 待判断的新记忆（共 ${matches.length} 条）
|
||||
|
||||
${newMemoriesText}
|
||||
|
||||
请逐条判断并输出决策 JSON 数组。当某条新记忆的候选列表为空时，该条直接输出 action=store。`;
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
/**
|
||||
* L1 Extraction Prompt: 情境切分 + 记忆提取
|
||||
*
|
||||
* Based on Kenty's validated prototype prompt (l1_memory_extraction_prompt.md).
|
||||
* System prompt handles scene segmentation + memory extraction in a single LLM call.
|
||||
* User prompt template fills in previous_scene_name, background_messages, new_messages.
|
||||
*/
|
||||
|
||||
import type { ConversationMessage } from "../conversation/l0-recorder.js";
|
||||
|
||||
// ============================
|
||||
// System Prompt
|
||||
// ============================
|
||||
|
||||
export const EXTRACT_MEMORIES_SYSTEM_PROMPT = `你是专业的"情境切分与记忆提取专家"。
|
||||
你的任务是分析用户的对话，判断情境切换，并从中提取结构化的核心记忆（仅限 persona, episodic, instruction 三类）。
|
||||
|
||||
### 任务一：情境切分（Scene Segmentation）
|
||||
分析【待提取的新消息】，结合【上一个情境】，判断并输出当前对话的情境。
|
||||
- 继承：无明显切换，沿用上一个情境。
|
||||
- 切换条件：用户发出明确指令（如"换话题"）、意图转变、或提出独立新目标。
|
||||
- 一段对话可能只有一个情境，也可能有多个情境（话题多次切换时）。
|
||||
- 命名规则："我（AI）在和xxx（用户身份）做xxx（目标活动）"（中文，30-50字，单句，全局唯一）。
|
||||
|
||||
---
|
||||
|
||||
### 任务二：核心记忆提取（Memory Extraction）
|
||||
结合背景和当前情境，仅从【待提取的新消息】中提取核心信息。
|
||||
|
||||
【通用提取原则】
|
||||
1. 宁缺毋滥：过滤琐碎闲聊、临时性指令和一次性操作（如"这次、本单"）；剔除不可靠的边缘信息。
|
||||
2. 独立完整：记忆必须"跳出当前对话依然成立"，无上下文也能看懂。提取主体必须以"用户（姓名）"或"AI"为核心。
|
||||
3. 归纳合并：强关联或因果关系的多条消息，必须合并为一条完整记忆，不可碎片化。
|
||||
|
||||
【支持提取的三大类型】（必须严格遵守类型规则）
|
||||
|
||||
1. 个性化记忆 (type: "persona")
|
||||
- 定义：用户的稳定属性、偏好、技能、价值观、习惯（如住所、职业、饮食禁忌）。
|
||||
- 提取句式："用户（[姓名]）喜欢/是/擅长..."
|
||||
- 打分 (priority)：80-100（健康/禁忌/核心特质）；50-70（一般喜好/技能）；<50（模糊次要，可丢弃）。
|
||||
- 触发词：喜欢、习惯、经常、我这个人...
|
||||
|
||||
2. 客观事件记忆 (type: "episodic")
|
||||
- 定义：客观发生的动作、决定、计划或达成结果。绝不包含纯主观感受。
|
||||
- 提取句式："用户（[姓名]）在 [最好是精确绝对时间] 于 [地点] [做了某事（可以包含起因、经过、结果）]"。
|
||||
- 时间约束：尽量基于消息的 timestamp 推算绝对时间，如能确定则在 metadata 中输出 activity_start_time 和 activity_end_time（ISO 8601格式）。无法确定时可省略。
|
||||
- 打分 (priority)：80-100（重要事件/计划）；60-70（一般完整活动）；<60（琐碎事项，直接丢弃）。
|
||||
|
||||
3. 全局指令记忆 (type: "instruction")
|
||||
- 定义：用户对 AI 提出的长期行为规则、格式偏好、语气控制。
|
||||
- 提取句式："用户要求/希望 AI 以后回答时..."
|
||||
- 触发词：以后都、从现在开始、记住、必须。
|
||||
- 打分 (priority)：-1（极其严格的全局死命令）；90-100（核心行为规则）；70-80（重要要求）；<70（临时要求，直接丢弃）。
|
||||
|
||||
---
|
||||
|
||||
### 不应该提取的内容
|
||||
- 琐碎闲聊、问候；临时性的纯工具性请求（如"这次帮我翻译一下"）
|
||||
- 一次性操作指令（如"这次、本单"相关）
|
||||
- 重复的内容；AI助手自身的行为或输出
|
||||
- 不属于以上3类的信息
|
||||
- 纯主观感受（不带客观事件的情绪表达）
|
||||
|
||||
---
|
||||
|
||||
### 任务三：输出格式规范（JSON）
|
||||
返回且仅返回一个合法的 JSON 数组。数组的每一项是一个情境，包含该情境的消息范围和抽取到的记忆：
|
||||
|
||||
[
|
||||
{
|
||||
"scene_name": "当前生成或继承的情境名称",
|
||||
"message_ids": ["属于该情境的消息ID列表"],
|
||||
"memories": [
|
||||
{
|
||||
"content": "完整、独立的记忆陈述（按对应类型的句式要求）",
|
||||
"type": "persona|episodic|instruction",
|
||||
"priority": 80,
|
||||
"source_message_ids": ["消息ID_1", "消息ID_2"],
|
||||
"metadata": {}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
|
||||
metadata 字段说明：
|
||||
- episodic 类型：如能确定活动时间，填入 {"activity_start_time": "ISO8601", "activity_end_time": "ISO8601"}
|
||||
- 其他类型或无法确定时间：输出空对象 {}
|
||||
|
||||
如果整段对话无有意义的记忆，也要输出情境分割结果，memories 为空数组：
|
||||
[
|
||||
{
|
||||
"scene_name": "情境名称",
|
||||
"message_ids": ["id1", "id2"],
|
||||
"memories": []
|
||||
}
|
||||
]
|
||||
|
||||
请严格按上述 JSON 数组格式输出，不要输出任何额外的 Markdown 代码块修饰符（如 \`\`\`json）或解释文本。`;
|
||||
|
||||
// ============================
|
||||
// Prompt Builder
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Format the user prompt for L1 extraction.
|
||||
*
|
||||
* @param newMessages - Messages to extract memories from (with ids and timestamps)
|
||||
* @param backgroundMessages - Previous messages for context only (not for extraction)
|
||||
* @param previousSceneName - The last known scene name (for continuity)
|
||||
*/
|
||||
export function formatExtractionPrompt(params: {
|
||||
newMessages: ConversationMessage[];
|
||||
backgroundMessages?: ConversationMessage[];
|
||||
previousSceneName?: string;
|
||||
}): string {
|
||||
const { newMessages, backgroundMessages = [], previousSceneName = "无" } = params;
|
||||
|
||||
const bgText = backgroundMessages.length > 0
|
||||
? backgroundMessages
|
||||
.map((m) => `[${m.id}] [${m.role}] [${new Date(m.timestamp).toISOString()}]: ${m.content}`)
|
||||
.join("\n\n")
|
||||
: "无";
|
||||
|
||||
const newText = newMessages
|
||||
.map((m) => `[${m.id}] [${m.role}] [${new Date(m.timestamp).toISOString()}]: ${m.content}`)
|
||||
.join("\n\n");
|
||||
|
||||
return `【上一个情境】：${previousSceneName}
|
||||
|
||||
【背景对话】（仅供理解上下文推断关系/时间，严禁从中提取记忆）：
|
||||
${bgText}
|
||||
|
||||
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||
|
||||
【待提取的新消息】（务必结合 timestamp 推算时间，只从这里提取记忆！）：
|
||||
${newText}`;
|
||||
}
|
||||
@@ -0,0 +1,189 @@
|
||||
/**
|
||||
* Persona Generation Prompt — instructs LLM to generate/update user persona
|
||||
* using the four-layer deep scan model.
|
||||
*
|
||||
* v3: Split into systemPrompt (role + constraints + logic + template) and
|
||||
* userPrompt (data). Tool names aligned to OpenClaw actual API (write/edit).
|
||||
*/
|
||||
|
||||
export interface PersonaPromptParams {
|
||||
mode: "first" | "incremental";
|
||||
currentTime: string;
|
||||
totalProcessed: number;
|
||||
sceneCount: number;
|
||||
changedSceneCount: number;
|
||||
changedScenesContent: string;
|
||||
existingPersona?: string;
|
||||
triggerInfo?: string;
|
||||
/** @deprecated Kept for call-site compatibility; no longer used in prompt. */
|
||||
personaFilePath: string;
|
||||
/** @deprecated Kept for call-site compatibility; no longer used in prompt. */
|
||||
checkpointPath: string;
|
||||
}
|
||||
|
||||
export interface PersonaPromptResult {
|
||||
systemPrompt: string;
|
||||
userPrompt: string;
|
||||
}
|
||||
|
||||
// ============================
|
||||
// System Prompt (stable: role + constraints + logic + template)
|
||||
// ============================
|
||||
|
||||
const PERSONA_SYSTEM_PROMPT = `# 🧬 Persona Architect - Incremental Evolution Protocol
|
||||
|
||||
请你结合已有的 persona.md 和新增/变化的 block 信息深度分析，然后使用文件工具将结果写入 \`persona.md\` 文件。
|
||||
|
||||
## ⛔ 文件操作约束（必须严格遵守）
|
||||
|
||||
1. **必须使用文件工具将最终 persona 内容写入 \`persona.md\`**。当前工作目录已设为数据目录，直接使用文件名 \`persona.md\`。
|
||||
- **首次生成 / 大幅重写**：使用 **write** 工具整体写入。参数：\`path\`=\`persona.md\`, \`content\`=完整内容
|
||||
- **增量更新（局部修改）**：使用 **edit** 工具精确替换。参数：\`path\`=\`persona.md\`, \`edits\`=[{\`oldText\`: 旧内容片段, \`newText\`: 新内容片段}]
|
||||
2. **只能操作 \`persona.md\` 这一个文件**，禁止读取或写入任何其他文件（包括 scene_blocks/、.metadata/ 等）。
|
||||
3. **写入的内容必须只包含最终的 persona 文档**，不要包含你的思考过程、分析步骤或任何非 persona 内容。
|
||||
4. **无需 read 工具**：当前 persona.md 的完整内容已在用户消息中提供，直接基于它进行更新即可。
|
||||
|
||||
### 🚫 严格禁止
|
||||
- **禁止过长**：persona.md 内容总长度不要超过 2000 字符，及时做总结和删除不重要的信息。
|
||||
- **禁止过度推测**：没提到的信息不要过度臆想导致产生幻觉，特别是在冷启动阶段，要保持克制，如果没有相关信息完全可以不填！
|
||||
- **禁止使用非场景来源的信息**：Persona 的所有内容必须且只能来自下方提供的场景数据。不要从 workspace 目录结构、文件路径、系统信息等技术元数据中提取任何关于用户的个人信息。
|
||||
- **禁止操作 persona.md 以外的任何文件**。
|
||||
|
||||
---
|
||||
|
||||
## ⚙️ 核心运作逻辑 (The Core Logic)
|
||||
|
||||
🧠 核心思维引擎：连接与综合 (Connect & Synthesize)
|
||||
请遵循 "叙事连贯性" 原则处理信息。禁止简单的罗列（No Bullet-point Spamming）。
|
||||
|
||||
1. 寻找"贯穿线" (The Connecting Thread)
|
||||
不要孤立地看信息。要寻找不同领域行为背后的共同逻辑。
|
||||
** 要保持精简，不过度猜想，如果不确定可以不写 **
|
||||
|
||||
执行以下**四层深度扫描**：
|
||||
|
||||
### 🟢 Layer 1: 基础锚点 (The Base & Facts) -> 【建立连接】
|
||||
* **扫描目标**: 确凿的事实、人口统计学特征、当前状态。
|
||||
* **实用价值**: 为 Agent 提供**破冰话题**和**上下文感知**。
|
||||
|
||||
### 🔵 Layer 2: 兴趣图谱 (The Interest Graph) -> 【提供谈资】
|
||||
* **扫描目标**: 用户投入时间、金钱或注意力的事物。
|
||||
* **提取原则**: **区分活跃度**（活跃爱好 / 被动消费 / 休眠兴趣）。
|
||||
* **实用价值**: 让 Agent 能够进行**高质量的闲聊 (Chit-chat)** 和 **生活推荐**。
|
||||
|
||||
### 🟡 Layer 3: 交互协议 (The Interface) -> 【消除摩擦】
|
||||
* **扫描目标**: 用户的沟通习惯、雷区、工作流偏好。
|
||||
* **实用价值**: 指导 Agent **如何说话、如何交付结果**，避免踩雷。
|
||||
|
||||
### 🔴 Layer 4: 认知内核 (The Core) -> 【深度共鸣】
|
||||
* **扫描目标**: 决策逻辑、矛盾点、终极驱动力。
|
||||
* **实用价值**: 让 Agent 成为**能够替用户做决策**的"副驾驶"。
|
||||
|
||||
---
|
||||
|
||||
## 📝 输出模板 (The Persona Template)
|
||||
|
||||
请参考以下格式，使用 **write** 工具写入最终内容。可以做自主调整（信息不足时可以减少或新增 chapter）（**必须保持 Markdown 格式**）：
|
||||
|
||||
\`\`\`\`markdown
|
||||
# User Narrative Profile
|
||||
|
||||
> **Archetype (核心原型)**: [一句话定义。例如：一位在现实重力下挣扎，但试图通过技术构建理想国的"务实理想主义者"。]
|
||||
|
||||
> **基本信息**
|
||||
（用户的基本信息，如年龄、性别、职业等，更新时若有冲突则覆盖，不冲突尽量叠加）
|
||||
-
|
||||
-
|
||||
|
||||
> **长期偏好**
|
||||
（你观察到的用户最稳定且可复用的偏好）
|
||||
-
|
||||
-
|
||||
|
||||
## 📖 Chapter 1: Context & Current State (全景语境)
|
||||
*(将基础事实与当前状态融合，写成一段连贯的背景介绍)*
|
||||
|
||||
**[这里写连贯描述，区别较大的时候可以分点阐述]**
|
||||
|
||||
## 🎨 Chapter 2: The Texture of Life (生活的肌理)
|
||||
*(将兴趣、消费、生活习惯串联起来，展示生活品味)*
|
||||
|
||||
**[这里写连贯的描述，重点在于"兴趣/偏好"和"品味"的统一性，区别较大的时候可以分点阐述]**
|
||||
|
||||
## 🤖 Chapter 3: Interaction & Cognitive Protocol (交互与认知协议)
|
||||
*(这是 Main Agent 的行动指南。为了实用，这里保持半结构化，但要解释"为什么")*
|
||||
|
||||
### 3.1 沟通策略 (How to Speak)
|
||||
### 3.2 决策逻辑 (How to Think)
|
||||
|
||||
## 🧩 Chapter 4: Deep Insights & Evolution (深层洞察与演变)
|
||||
*(人类学观察笔记)*
|
||||
|
||||
* **矛盾统一性**: [描述用户身上看似冲突但实则合理的特质]。
|
||||
* **演变轨迹**: [可加上时间，分为多点，描述用户最近发生的变化]。
|
||||
* **涌现特征**: 提炼 3-7 个最核心的特质标签，每个标签单独一行并附上简短注释（10-15字）
|
||||
- \`TagName\` - 简短注释说明
|
||||
\`\`\`\`
|
||||
|
||||
---
|
||||
|
||||
### ⚠️ 成功标准
|
||||
- ✅ **必须使用 write 或 edit 工具写入最终结果到 \`persona.md\`**
|
||||
- ✅ 基于场景证据生成深度洞察
|
||||
- ✅ 内容到 Chapter 4 结束（不包含场景导航，工程会自动追加）
|
||||
- ✅ 必须严格按照上面的模板格式
|
||||
- ✅ 不要添加场景导航（工程会自动追加）
|
||||
- ✅ 只操作 persona.md，不要操作其他文件`;
|
||||
|
||||
// ============================
|
||||
// User Prompt builder (dynamic data)
|
||||
// ============================
|
||||
|
||||
export function buildPersonaPrompt(params: PersonaPromptParams): PersonaPromptResult {
|
||||
const {
|
||||
mode,
|
||||
currentTime,
|
||||
totalProcessed,
|
||||
sceneCount,
|
||||
changedSceneCount,
|
||||
changedScenesContent,
|
||||
existingPersona,
|
||||
triggerInfo,
|
||||
} = params;
|
||||
|
||||
const modeLabel = mode === "first" ? "🆕 首次生成" : "🔄 迭代更新";
|
||||
|
||||
const triggerSection = triggerInfo
|
||||
? `\n### 触发信息\n${triggerInfo}\n`
|
||||
: "";
|
||||
|
||||
const existingPersonaSection = existingPersona
|
||||
? `\n## 📄 当前 Persona（工程已预加载）\n\n` +
|
||||
`*以下是现有 persona.md 的完整内容（${existingPersona.length} 字符），基于此更新后请控制在2000字内：*\n\n` +
|
||||
`\`\`\`markdown\n${existingPersona}\n\`\`\`\n\n---\n`
|
||||
: "";
|
||||
|
||||
const iterationGuide = mode === "incremental"
|
||||
? `\n## 🔄 迭代决策指南\n\n` +
|
||||
`面对变化场景，自主判断处理方式：强化（佐证已有洞察）/ 补充（新维度）/ 修正（矛盾）/ 重构（结构调整）/ 不改（无有用新增内容）。\n`
|
||||
: "";
|
||||
|
||||
const userPrompt = `**⏰ 更新时间**: ${currentTime}
|
||||
**模式**: ${modeLabel}
|
||||
${triggerSection}
|
||||
## 📊 统计
|
||||
- **总记忆数**: ${totalProcessed} 条
|
||||
- **场景总数**: ${sceneCount} 个
|
||||
- **变化场景**: ${changedSceneCount} 个（自上次更新后）
|
||||
|
||||
---
|
||||
${changedScenesContent}
|
||||
|
||||
${existingPersonaSection}
|
||||
${iterationGuide}`;
|
||||
|
||||
return {
|
||||
systemPrompt: PERSONA_SYSTEM_PROMPT,
|
||||
userPrompt,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,263 @@
|
||||
/**
|
||||
* Scene Extraction Prompt — instructs LLM to consolidate memories into scene blocks
|
||||
* using file tools (read, write, edit).
|
||||
*
|
||||
* v2: Split into systemPrompt (role + constraints + workflow + output spec) and
|
||||
* userPrompt (dynamic data). Tool names aligned to OpenClaw actual API.
|
||||
*
|
||||
* Scene files can be updated via:
|
||||
* - read + write (full rewrite) for large structural changes
|
||||
* - edit (targeted partial updates, e.g. updating a single section)
|
||||
*
|
||||
* Security: The LLM is sandboxed to scene_blocks/ only (workspaceDir = scene_blocks/).
|
||||
* It has NO visibility into checkpoint, scene_index, persona.md, or any other system file.
|
||||
* File deletion is achieved via "soft-delete" — writing the marker `[DELETED]` to the file
|
||||
* — and the SceneExtractor subsequently removes soft-deleted files with fs.unlink.
|
||||
* Note: writing an empty/whitespace-only string is rejected by the core write tool's
|
||||
* parameter validation, so we use a non-empty marker instead.
|
||||
*
|
||||
* Persona update requests are communicated via text output signals (out-of-band),
|
||||
* parsed by the engineering side after LLM execution completes.
|
||||
*/
|
||||
|
||||
export interface SceneExtractionPromptParams {
|
||||
memoriesJson: string;
|
||||
sceneSummaries: string;
|
||||
currentTimestamp: string;
|
||||
sceneCountWarning?: string;
|
||||
/** List of existing scene filenames (relative, e.g. ["work.md", "hobby.md"]) */
|
||||
existingSceneFiles?: string[];
|
||||
/** Maximum number of scene blocks allowed */
|
||||
maxScenes: number;
|
||||
}
|
||||
|
||||
export interface SceneExtractionPromptResult {
|
||||
systemPrompt: string;
|
||||
userPrompt: string;
|
||||
}
|
||||
|
||||
// ============================
|
||||
// System Prompt builder (role + constraints + workflow + output spec)
|
||||
// Contains maxScenes as a constraint parameter.
|
||||
// ============================
|
||||
|
||||
function buildSceneSystemPrompt(maxScenes: number): string {
|
||||
return `# Memory Consolidation Architect
|
||||
|
||||
## 角色定义 (Role Definition)
|
||||
你是记忆整合架构师。你的目标是为用户构建一个"数字第二大脑"。你不仅仅是在记录数据，你更像是一位人类学家和心理学家，负责分析原始记忆，从中提取核心特征、捕捉隐性信号，并构建不断演变的叙事。
|
||||
|
||||
|
||||
## 架构模型
|
||||
|
||||
### Layer 1 (Input): Raw Memories
|
||||
- **来源**：API 分批召回（每批 20 条）
|
||||
- **状态**：碎片化、无序
|
||||
|
||||
### Layer 2 (Processing): Scene Diaries
|
||||
- **形态**：**不是清单，是连贯的叙事文档**
|
||||
- **逻辑**：将 L1 碎片融合进特定场景文件
|
||||
- **动作**：Create（创建）、Integrate（整合）、Rewrite（重写）
|
||||
- **禁止**：简单追加列表
|
||||
|
||||
你主要负责L1到L2的生成任务
|
||||
|
||||
## 输入环境 (Input Context)
|
||||
你将接收三个输入：
|
||||
1. 新增记忆 (New Memory): 一段原始的、非结构化的新近回忆信息。
|
||||
2. 现有 Block 映射表 (Existing Blocks Map): 包含当前所有记忆块（Markdown 文件）的文件名和摘要的列表。
|
||||
3. 当前时间 (Current Time): 用于生成元数据的具体时间戳。
|
||||
|
||||
**⚠️ 场景文件数量上限：${maxScenes} 个。处理完成后目录中的场景文件数量必须严格小于此上限。**
|
||||
|
||||
## ⛔ 文件操作约束（必须严格遵守）
|
||||
1. **所有文件操作使用相对文件名**（如 \`技术研究-Rust学习.md\`），当前工作目录已设为场景文件目录
|
||||
2. **read 只能读取用户消息中"已有场景文件清单"列出的文件**，禁止猜测或编造不在清单中的文件名
|
||||
3. **创建新场景文件时**，使用 **write** 工具。参数：\`path\`=文件名, \`content\`=完整内容
|
||||
4. **局部更新场景文件**：使用 **edit** 工具。参数：\`path\`=文件名, \`edits\`=[{\`oldText\`: 旧内容, \`newText\`: 新内容}]。对于大范围重写或结构性变更，建议使用 **read** + **write** 整体重写。
|
||||
5. **场景索引和系统配置由工程系统自动维护**，你只需专注于操作 \`.md\` 场景文件
|
||||
6. **删除文件的唯一方式**：使用 **write** 工具将文件内容写为 \`[DELETED]\` 标记（\`path\`=文件名, \`content\`=\`[DELETED]\`）。系统会自动清理带有此标记的文件。**禁止**写入空字符串（会被系统拒绝）。**禁止**用 \`[ARCHIVE]\`、\`[CONSOLIDATED]\` 等其他标记替代删除——只有 \`[DELETED]\` 标记会触发系统清理。
|
||||
7. **禁止创建报告/整合/汇总类文件**。你的输出必须是有意义的场景叙事文件（如"技术架构与工程实践.md"、"日常生活与工作节奏.md"）。禁止创建以 BATCH、REPORT、CONSOLIDATION、INTEGRATION、ARCHIVE、SUMMARY 等为前缀的文件。
|
||||
|
||||
## 工作流与逻辑 (Workflow & Logic)
|
||||
在生成输出之前，你必须执行以下"思维链"过程：
|
||||
|
||||
### ⚠️ 阶段 0：强制检查场景总数（必须先执行）
|
||||
|
||||
**在处理任何记忆之前，你必须：**
|
||||
|
||||
1. **统计当前场景总数**：查看 "Existing Scene Blocks Summary" 顶部标注的当前场景总数
|
||||
2. **最终目标**：处理完成后，目录中的场景文件数量必须 **严格小于 ${maxScenes}**
|
||||
3. **遵守分级预警**：
|
||||
- 红色预警（≥ ${maxScenes}）：**必须先通过 MERGE 减少文件数量**，将最相似的 2-4 个场景合并为 1 个，**并删除被合并的旧文件**，直到文件数 < ${maxScenes} 后，再处理新记忆
|
||||
- 橙色预警（= ${maxScenes - 1}）：**只能 UPDATE 现有场景，不能 CREATE 新场景**
|
||||
- 黄色预警（接近 ${maxScenes}）：**优先 UPDATE 或主动 MERGE 相似场景**
|
||||
|
||||
**合并优先级**（当需要合并时，按以下顺序选择）：
|
||||
1. **主题高度重叠**：如"Python后端开发"和"Go后端开发" → 合并为"后端开发技术栈"
|
||||
2. **叙事弧线相同**：如"求职材料-JD匹配"和"职业发展-能力对齐" → 合并为"职业发展与求职"
|
||||
3. **热度最低的场景**：如果没有明显重叠，合并或删除 heat 最低的 2-3 个场景
|
||||
|
||||
### 阶段 1：分析与分类
|
||||
分析 新增记忆。它的核心领域是什么？（例如：编程风格、情绪状态、职业轨迹、人际关系）。
|
||||
提取事实事件链（触发 -> 行动 -> 结果）以及底层的心理状态。
|
||||
|
||||
### 阶段 2：检索与策略选择
|
||||
将新记忆与 现有 Block 映射表 进行比对。
|
||||
需要时使用 **read** 工具读取完整场景文件内容
|
||||
**只能读取用户消息中"已有场景文件清单"列出的文件，禁止猜测其他文件路径。**
|
||||
|
||||
**核心原则：默认策略是 UPDATE，不是 CREATE。** 当犹豫于 UPDATE 和 CREATE 之间时，选择 UPDATE。
|
||||
|
||||
策略选择（按优先级排序）：
|
||||
1. **UPDATE（更新）**【首选策略】: 如果存在相关的 Block（基于摘要或文件名的相似性），先用 **read** 读取文件内的具体信息，再锁定该 Block 进行更新（**write** 整体重写 或 **edit** 局部替换）
|
||||
2. **MERGE（合并）**:
|
||||
- 合并的新 block 应该是生成概括性更强的场景，包含已有的多个相似场景
|
||||
- **强制合并**：当前 Block 总数 **≥ ${maxScenes}** 时，必须先将多个相似记忆合并
|
||||
- **主动合并**：即使未达上限，如果两个 Block 属于同一叙事弧线，也应合并以增加深度
|
||||
- **⚠️ 合并后必须删除旧文件**：被合并的旧场景文件必须通过 **write** 写入 \`[DELETED]\` 标记。**仅仅打标记（如 [ARCHIVE]、[CONSOLIDATED]）不算删除，文件仍会占用配额。**
|
||||
3. **CREATE（新建）**【最后手段】:
|
||||
- **前提条件**：当前场景总数 < ${maxScenes}
|
||||
- **CREATE 前的强制验证**：必须先用 **read** 检查至少 2 个最相似的现有场景，确认新记忆确实无法融入后才能 CREATE。跳过验证直接 CREATE 是被禁止的
|
||||
- 如果话题是全新的且与现有内容区分度高，可以创建新 Block
|
||||
- **每次批处理最多新增 1 个场景**
|
||||
|
||||
**示例 A：新记忆整合进已有 block（UPDATE - 原地更新）**
|
||||
**具体操作步骤（工具调用）**：
|
||||
1. **read**(\`path\`='Python后端开发.md') → 获取已有内容 A
|
||||
2. 分析新记忆 + 已有内容 A → 整合生成新内容 B（\`heat = 旧heat + 1\`）
|
||||
3. **write**(\`path\`='Python后端开发.md', \`content\`=B) → **整体重写该场景文件**
|
||||
或 **edit**(\`path\`='Python后端开发.md', \`edits\`=[{\`oldText\`: 旧章节, \`newText\`: 新章节}]) → **局部更新某部分**
|
||||
|
||||
**示例 B：合并多个 block（MERGE — 合并后必须删除旧文件）**
|
||||
**具体操作步骤（工具调用）**：
|
||||
1. **read**(\`path\`='Python后端开发.md') → 获取内容 A
|
||||
2. **read**(\`path\`='Go后端开发.md') → 获取内容 B
|
||||
3. 整合 A + B + 新记忆 → 生成新内容 C（\`heat = heatA + heatB + 1\`）
|
||||
4. **write**(\`path\`='后端开发技术栈.md', \`content\`=C) → 创建合并后的新文件
|
||||
5. **write**(\`path\`='Python后端开发.md', \`content\`='[DELETED]') → **⚠️ 删除旧文件 A**
|
||||
6. **write**(\`path\`='Go后端开发.md', \`content\`='[DELETED]') → **⚠️ 删除旧文件 B**
|
||||
**关键**：步骤 5-6 是必须的！不执行删除 = 文件总数不减少 = 合并无效。
|
||||
|
||||
### 阶段 3：撰写与合成（核心任务）
|
||||
深度整合: 严禁简单的文本追加。你必须结合上下文（基于摘要或提供的原始内容）重写叙事，将新信息自然地融入其中。
|
||||
隐性推断: 寻找用户 没说出口 的信息。更新"隐性信号"部分。
|
||||
冲突检测: 如果新记忆与旧记忆相矛盾，将其记录在"演变轨迹"或"待确认/矛盾点"中。
|
||||
|
||||
### 撰写准则 (严格遵守)
|
||||
核心部分禁止列表: "用户核心特征"和"核心叙事"必须是连贯的段落，信息要连贯，可以分段。
|
||||
叙事弧线: "核心叙事"必须遵循故事结构（情境 -> 行动 -> 结果）。
|
||||
|
||||
### 热度管理 (Heat Management):
|
||||
新建 Block: heat: 1
|
||||
更新 Block: heat: 旧heat + 1
|
||||
合并 Block: heat: sum(所有相关block的heat) + 1
|
||||
|
||||
## 输出规范 (Output Specification)
|
||||
|
||||
### 📄 场景文件内容（必须输出）
|
||||
|
||||
请你参考这个模板输出 .md 文件的内容或基于已有md进行更新，每个md控制在1500字符内。不要把模板本身放在 Markdown 代码块中，只需直接输出要写入文件的原始文本。
|
||||
|
||||
\`\`\`markdown
|
||||
-----META-START-----
|
||||
created: {{EXISTING_CREATED_TIME_OR_CURRENT_TIME}}
|
||||
updated: {{CURRENT_TIME}}
|
||||
summary: [30-40 words concise summary for indexing]
|
||||
heat: [Integer]
|
||||
-----META-END-----
|
||||
|
||||
## 用户基础信息
|
||||
[可为空，如果没有可不写这节，可按照需求添加更多点，合并和更新方式尽量叠加，有冲突则覆盖]
|
||||
-姓名：
|
||||
-职业：
|
||||
-居住地：
|
||||
- ……
|
||||
|
||||
## 用户核心特征
|
||||
[这里不是列表！是一段连贯的描述。你细心推断出来最核心的用户特征，宁缺毋滥，**控制在100字以内**]
|
||||
[示例: 用户在后端开发方面表现出对 Python 的强烈偏好，特别是异步框架。近期（2026-02）开始关注 Rust 的所有权机制，这表明用户有向系统级编程转型的意图。]
|
||||
|
||||
## 用户偏好
|
||||
[这里可以是列表！**如果没有可以为不写这节**，记录用户明确的偏好信息（显性偏好），注意不要重复信息，不要流水账，偏好要可复用，更新时可以动态整合甚至重写]
|
||||
[示例：用户喜欢吃苹果]
|
||||
|
||||
## 隐性信号
|
||||
[这是给人类学家看的，记录那些"没明说但很重要"的事，和显性偏好不一样，一定是你推断出来的，需要深思熟虑后再生成，可以为空，宁缺毋滥。你可以随时更新/删除/修改这里的信息]
|
||||
|
||||
## 核心叙事
|
||||
[这里不是列表！是一段连贯的描述，**控制在400字以内**，注意不要重复信息，不要流水账，可以动态整合甚至重写]
|
||||
*(这里记录连贯的故事，必须包含 Trigger -> Action -> Result)*
|
||||
|
||||
[ 示例：本周用户主要集中在后端重构上。初期因为旧代码的耦合度高感到沮丧（**情绪点**），但他拒绝了"打补丁"的建议，坚持进行彻底解耦（**决策点**）。他在此过程中频繁查阅架构设计模式，表现出对"代码洁癖"的执着。]
|
||||
|
||||
|
||||
## 演变轨迹
|
||||
> [注意] 可以为空，仅记录【用户偏好/性格/重大观念】转变，不记录琐碎、日常更新。当发生冲突时，不要直接覆盖，要记录变化轨迹。
|
||||
- [2026-01-10]: 从 "反对加班" 转向 "接受弹性工作"，原因：创业压力（记忆ID: #987）
|
||||
|
||||
|
||||
## 待确认/矛盾点
|
||||
- [记录当前无法整合的矛盾信息，等待未来记忆澄清]
|
||||
|
||||
\`\`\`
|
||||
|
||||
|
||||
|
||||
#### 主动触发 Persona 更新（可选）
|
||||
|
||||
**触发条件**：重大价值观转变、跨场景突破性洞察。
|
||||
|
||||
**触发方式**：在你的 text output 中输出以下标记（不是文件操作）：
|
||||
|
||||
[PERSONA_UPDATE_REQUEST]
|
||||
reason: 具体原因描述
|
||||
[/PERSONA_UPDATE_REQUEST]
|
||||
|
||||
|
||||
**执行文件操作**（必须使用工具）：
|
||||
- 使用 **read** 读取需要更新的场景文件
|
||||
- 使用 **write** 创建新文件或**整体重写**已有场景文件
|
||||
- 使用 **edit** 对场景文件进行**局部更新**（如只更新某个章节）
|
||||
- **删除文件**：使用 **write**(\`path\`=文件名, \`content\`='[DELETED]') 写入删除标记。系统会自动清理这些文件。**重要**：只有 \`[DELETED]\` 标记会触发系统清理。写入空字符串会被系统拒绝，写入 \`[ARCHIVE]\`、\`[CONSOLIDATED]\` 等标记**不会删除文件**，文件会继续占用场景配额。`;
|
||||
}
|
||||
|
||||
// ============================
|
||||
// User Prompt builder (dynamic data)
|
||||
// ============================
|
||||
|
||||
export function buildSceneExtractionPrompt(params: SceneExtractionPromptParams): SceneExtractionPromptResult {
|
||||
const {
|
||||
memoriesJson,
|
||||
sceneSummaries,
|
||||
currentTimestamp,
|
||||
sceneCountWarning,
|
||||
existingSceneFiles,
|
||||
maxScenes,
|
||||
} = params;
|
||||
|
||||
const warningSection = sceneCountWarning
|
||||
? `\n⚠️ **场景数量警告**: ${sceneCountWarning}\n`
|
||||
: "";
|
||||
|
||||
const fileListSection = existingSceneFiles && existingSceneFiles.length > 0
|
||||
? `### 📁 已有场景文件清单（仅以下文件可 read）\n${existingSceneFiles.map((f) => `- \`${f}\``).join("\n")}\n`
|
||||
: `### 📁 已有场景文件清单\n（当前无已有场景文件）\n`;
|
||||
|
||||
const userPrompt = `${warningSection}
|
||||
### 1️⃣ New Memories List
|
||||
${memoriesJson}
|
||||
|
||||
### 2️⃣ Existing Scene Blocks Summary
|
||||
${sceneSummaries}
|
||||
|
||||
### 3️⃣ Current Timestamp
|
||||
${currentTimestamp}
|
||||
|
||||
${fileListSection}`;
|
||||
|
||||
return {
|
||||
systemPrompt: buildSceneSystemPrompt(maxScenes),
|
||||
userPrompt,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,405 @@
|
||||
/**
|
||||
* L1 Memory Conflict Detection (Batch Mode): decides how to handle multiple new
|
||||
* memories against existing records in a single LLM call.
|
||||
*
|
||||
* v4: Removed JSONL-based Jaccard fallback. Candidate recall now relies exclusively
|
||||
* on vector search (primary) and FTS5 BM25 (degraded). If neither is available,
|
||||
* conflict detection is skipped entirely — all memories go straight to store.
|
||||
*
|
||||
* Two-phase approach:
|
||||
* 1. Candidate search per new memory — vector recall or FTS5 keyword recall (fast, no LLM)
|
||||
* 2. Batch LLM judgment on all new memories + their candidate pools (single call)
|
||||
*/
|
||||
|
||||
import type { ExtractedMemory, MemoryRecord, DedupDecision, MemoryType } from "./l1-writer.js";
|
||||
import { CONFLICT_DETECTION_SYSTEM_PROMPT, formatBatchConflictPrompt } from "../prompts/l1-dedup.js";
|
||||
import type { CandidateMatch } from "../prompts/l1-dedup.js";
|
||||
import { CleanContextRunner } from "../../utils/clean-context-runner.js";
|
||||
import { sanitizeJsonForParse } from "../../utils/sanitize.js";
|
||||
import type { IMemoryStore } from "../store/types.js";
|
||||
import { buildFtsQuery } from "../store/sqlite.js";
|
||||
import type { EmbeddingService } from "../store/embedding.js";
|
||||
import type { LLMRunner } from "../types.js";
|
||||
|
||||
interface Logger {
|
||||
debug?: (message: string) => void;
|
||||
info: (message: string) => void;
|
||||
warn: (message: string) => void;
|
||||
error: (message: string) => void;
|
||||
}
|
||||
|
||||
const TAG = "[memory-tdai][l1-dedup]";
|
||||
|
||||
// ============================
|
||||
// Core function (batch mode)
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Batch conflict detection: compare all new memories against existing records
|
||||
* in a single LLM call.
|
||||
*
|
||||
* Candidate recall strategy (3-tier degradation):
|
||||
* 1. Vector recall (vectorStore + embeddingService) — cosine similarity (best)
|
||||
* 2. FTS5 keyword recall (vectorStore with FTS available) — BM25 ranking (degraded)
|
||||
* 3. Skip conflict detection entirely — all memories go straight to "store"
|
||||
*
|
||||
* The old JSONL-based Jaccard fallback has been removed. If neither vector search
|
||||
* nor FTS is available, we skip dedup rather than paying the O(N) full-file-scan cost.
|
||||
*
|
||||
* @param memories - Newly extracted memories (with record_id)
|
||||
* @param config - OpenClaw config (for LLM access)
|
||||
* @param logger - Optional logger
|
||||
* @param model - Optional model override
|
||||
* @param vectorStore - Optional vector store for cosine similarity search
|
||||
* @param embeddingService - Optional embedding service for computing query vectors
|
||||
* @param conflictRecallTopK - Top-K candidates to recall per new memory (default: 5)
|
||||
* @returns Array of dedup decisions, one per new memory
|
||||
*/
|
||||
export async function batchDedup(params: {
|
||||
memories: Array<ExtractedMemory & { record_id: string }>;
|
||||
config: unknown;
|
||||
logger?: Logger;
|
||||
model?: string;
|
||||
/** Vector store for cosine similarity candidate recall */
|
||||
vectorStore?: IMemoryStore;
|
||||
/** Embedding service for computing query vectors */
|
||||
embeddingService?: EmbeddingService;
|
||||
/** Top-K candidates per new memory (default: 5) */
|
||||
conflictRecallTopK?: number;
|
||||
/** Override embedding timeout for capture-path calls (milliseconds) */
|
||||
embeddingTimeoutMs?: number;
|
||||
/** Host-neutral LLM runner — when provided, used instead of CleanContextRunner. */
|
||||
llmRunner?: LLMRunner;
|
||||
}): Promise<DedupDecision[]> {
|
||||
const { memories, config, logger, model, vectorStore, embeddingService, llmRunner } = params;
|
||||
const topK = params.conflictRecallTopK ?? 5;
|
||||
|
||||
if (memories.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const storeAll = () =>
|
||||
memories.map((m) => ({
|
||||
record_id: m.record_id,
|
||||
action: "store" as const,
|
||||
target_ids: [],
|
||||
}));
|
||||
|
||||
// Determine what recall capabilities are available
|
||||
const hasVectorData = vectorStore && (await vectorStore.countL1()) > 0;
|
||||
const hasFts = vectorStore?.isFtsAvailable() ?? false;
|
||||
|
||||
// Fast path: no recall capability at all → skip dedup
|
||||
if (!hasVectorData && !hasFts) {
|
||||
logger?.debug?.(`${TAG} No vector data and no FTS available, skipping conflict detection for ${memories.length} memories`);
|
||||
return storeAll();
|
||||
}
|
||||
|
||||
// Phase 1: Find candidates
|
||||
//
|
||||
// Decision tree (after the fast-path guard above, vectorStore is guaranteed non-null):
|
||||
// hasVectorData + embeddingService → Tier 1 vector recall (FTS fallback on error)
|
||||
// otherwise hasFts → Tier 2 FTS keyword recall
|
||||
// otherwise → skip dedup (defensive; shouldn't reach here)
|
||||
let matches: CandidateMatch[];
|
||||
|
||||
if (hasVectorData && embeddingService) {
|
||||
// === Tier 1: Vector recall mode ===
|
||||
logger?.debug?.(`${TAG} Using vector recall mode (topK=${topK})`);
|
||||
try {
|
||||
matches = await findCandidatesByVector(memories, vectorStore!, embeddingService, topK, logger, params.embeddingTimeoutMs);
|
||||
} catch (err) {
|
||||
logger?.warn?.(
|
||||
`${TAG} Vector recall failed, falling back to FTS keyword: ${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
// Degrade to FTS keyword recall
|
||||
if (hasFts) {
|
||||
matches = await findCandidatesByFts(memories, vectorStore!, logger);
|
||||
} else {
|
||||
logger?.debug?.(`${TAG} FTS not available either, skipping conflict detection`);
|
||||
return storeAll();
|
||||
}
|
||||
}
|
||||
} else if (hasFts) {
|
||||
// === Tier 2: FTS keyword recall ===
|
||||
logger?.debug?.(`${TAG} Using FTS keyword recall mode (no embedding service or no vector data)`);
|
||||
matches = await findCandidatesByFts(memories, vectorStore!, logger);
|
||||
} else {
|
||||
// Shouldn't reach here given the fast-path check above, but be defensive
|
||||
logger?.debug?.(`${TAG} No usable recall path, skipping conflict detection`);
|
||||
return storeAll();
|
||||
}
|
||||
|
||||
// Check if any memory has candidates
|
||||
const hasAnyCandidates = matches.some((m) => m.candidates.length > 0);
|
||||
|
||||
if (!hasAnyCandidates) {
|
||||
logger?.debug?.(`${TAG} No similar records found for any memory, all will be stored`);
|
||||
return storeAll();
|
||||
}
|
||||
|
||||
// Phase 2: Batch LLM judgment
|
||||
return runLlmJudgment(matches, memories, config, logger, model, llmRunner);
|
||||
}
|
||||
|
||||
/**
|
||||
* Phase 2: Run batch LLM judgment on candidate matches.
|
||||
*/
|
||||
async function runLlmJudgment(
|
||||
matches: CandidateMatch[],
|
||||
memories: Array<ExtractedMemory & { record_id: string }>,
|
||||
config: unknown,
|
||||
logger: Logger | undefined,
|
||||
model: string | undefined,
|
||||
llmRunner?: LLMRunner,
|
||||
): Promise<DedupDecision[]> {
|
||||
logger?.debug?.(`${TAG} Running batch conflict detection for ${memories.length} memories`);
|
||||
|
||||
try {
|
||||
const userPrompt = formatBatchConflictPrompt(matches);
|
||||
let result: string;
|
||||
|
||||
if (llmRunner) {
|
||||
// Use the host-neutral LLMRunner interface
|
||||
result = await llmRunner.run({
|
||||
prompt: userPrompt,
|
||||
systemPrompt: CONFLICT_DETECTION_SYSTEM_PROMPT,
|
||||
taskId: "l1-conflict-detection",
|
||||
timeoutMs: 180_000,
|
||||
});
|
||||
} else {
|
||||
// Fallback: create CleanContextRunner (OpenClaw path)
|
||||
const runner = new CleanContextRunner({
|
||||
config,
|
||||
modelRef: model,
|
||||
enableTools: false,
|
||||
logger,
|
||||
});
|
||||
|
||||
result = await runner.run({
|
||||
prompt: userPrompt,
|
||||
systemPrompt: CONFLICT_DETECTION_SYSTEM_PROMPT,
|
||||
taskId: "l1-conflict-detection",
|
||||
timeoutMs: 180_000,
|
||||
});
|
||||
}
|
||||
|
||||
const decisions = parseBatchResult(result, memories, logger);
|
||||
return decisions;
|
||||
} catch (err) {
|
||||
logger?.warn?.(
|
||||
`${TAG} Batch conflict detection failed, defaulting all to store: ${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
return memories.map((m) => ({
|
||||
record_id: m.record_id,
|
||||
action: "store" as const,
|
||||
target_ids: [],
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Candidate recall strategies
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Vector-based candidate recall (aligned with prototype):
|
||||
* batch-embed new memories → cosine search in VectorStore → exclude self-batch → return candidates.
|
||||
*/
|
||||
async function findCandidatesByVector(
|
||||
memories: Array<ExtractedMemory & { record_id: string }>,
|
||||
vectorStore: IMemoryStore,
|
||||
embeddingService: EmbeddingService,
|
||||
topK: number,
|
||||
logger?: Logger,
|
||||
embeddingTimeoutMs?: number,
|
||||
): Promise<CandidateMatch[]> {
|
||||
const newRecordIds = new Set(memories.map((m) => m.record_id));
|
||||
|
||||
// Batch-compute embeddings for all new memories
|
||||
const texts = memories.map((m) => m.content);
|
||||
const embeddings = await embeddingService.embedBatch(texts, embeddingTimeoutMs ? { timeoutMs: embeddingTimeoutMs } : undefined);
|
||||
|
||||
const matches: CandidateMatch[] = [];
|
||||
|
||||
for (let i = 0; i < memories.length; i++) {
|
||||
const mem = memories[i];
|
||||
const queryVec = embeddings[i];
|
||||
|
||||
// Vector search top-K (request extra to account for self-batch filtering)
|
||||
const searchResults = await vectorStore.searchL1Vector(queryVec, topK + memories.length, mem.content);
|
||||
|
||||
// Exclude records from current batch, convert to MemoryRecord format
|
||||
const candidates: MemoryRecord[] = searchResults
|
||||
.filter((r) => !newRecordIds.has(r.record_id))
|
||||
.slice(0, topK)
|
||||
.map((r) => ({
|
||||
id: r.record_id,
|
||||
content: r.content,
|
||||
type: r.type as MemoryRecord["type"],
|
||||
priority: r.priority,
|
||||
scene_name: r.scene_name,
|
||||
source_message_ids: [],
|
||||
metadata: {},
|
||||
timestamps: [r.timestamp_str].filter(Boolean),
|
||||
createdAt: "",
|
||||
updatedAt: "",
|
||||
sessionKey: r.session_key,
|
||||
sessionId: r.session_id,
|
||||
}));
|
||||
|
||||
matches.push({ newMemory: mem, candidates });
|
||||
}
|
||||
|
||||
logger?.debug?.(
|
||||
`${TAG} Vector recall: ${matches.map((m) => `${m.newMemory.record_id}→${m.candidates.length}`).join(", ")}`,
|
||||
);
|
||||
|
||||
return matches;
|
||||
}
|
||||
|
||||
/**
|
||||
* FTS5-based candidate recall:
|
||||
* Uses the FTS index for efficient BM25-ranked keyword matching.
|
||||
* This replaces the old Jaccard word-overlap fallback entirely.
|
||||
*/
|
||||
async function findCandidatesByFts(
|
||||
memories: Array<ExtractedMemory & { record_id: string }>,
|
||||
vectorStore: IMemoryStore,
|
||||
_logger?: Logger,
|
||||
): Promise<CandidateMatch[]> {
|
||||
const newRecordIds = new Set(memories.map((m) => m.record_id));
|
||||
const matches: CandidateMatch[] = [];
|
||||
|
||||
for (const mem of memories) {
|
||||
const ftsQuery = buildFtsQuery(mem.content);
|
||||
if (ftsQuery) {
|
||||
const ftsResults = await vectorStore.searchL1Fts(ftsQuery, 10);
|
||||
// Filter out records from the current batch
|
||||
const candidates: MemoryRecord[] = ftsResults
|
||||
.filter((r) => !newRecordIds.has(r.record_id))
|
||||
.slice(0, 5)
|
||||
.map((r) => ({
|
||||
id: r.record_id,
|
||||
content: r.content,
|
||||
type: r.type as MemoryRecord["type"],
|
||||
priority: r.priority,
|
||||
scene_name: r.scene_name,
|
||||
source_message_ids: [],
|
||||
metadata: r.metadata_json ? (() => { try { return JSON.parse(r.metadata_json); } catch { return {}; } })() : {},
|
||||
timestamps: [r.timestamp_str].filter(Boolean),
|
||||
createdAt: "",
|
||||
updatedAt: "",
|
||||
sessionKey: r.session_key,
|
||||
sessionId: r.session_id,
|
||||
}));
|
||||
matches.push({ newMemory: mem, candidates });
|
||||
} else {
|
||||
matches.push({ newMemory: mem, candidates: [] });
|
||||
}
|
||||
}
|
||||
|
||||
_logger?.debug?.(`${TAG} FTS keyword recall: ${matches.map((m) => `${m.newMemory.record_id}→${m.candidates.length}`).join(", ")}`);
|
||||
return matches;
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Result parsing
|
||||
// ============================
|
||||
|
||||
const VALID_TYPES: MemoryType[] = ["persona", "episodic", "instruction"];
|
||||
|
||||
/**
|
||||
* Parse the LLM's batch conflict detection JSON response.
|
||||
*
|
||||
* Expected format: [{record_id, action, target_ids, merged_content, merged_type, merged_priority, merged_timestamps}]
|
||||
*/
|
||||
function parseBatchResult(
|
||||
raw: string,
|
||||
memories: Array<ExtractedMemory & { record_id: string }>,
|
||||
logger?: Logger,
|
||||
): DedupDecision[] {
|
||||
try {
|
||||
// Strip markdown code block wrappers
|
||||
let cleaned = raw.trim();
|
||||
if (cleaned.startsWith("```")) {
|
||||
cleaned = cleaned.replace(/^```(?:json)?\s*\n?/, "").replace(/\n?```\s*$/, "");
|
||||
}
|
||||
|
||||
// Extract JSON array
|
||||
const arrayMatch = cleaned.match(/\[[\s\S]*\]/);
|
||||
if (!arrayMatch) {
|
||||
logger?.warn?.(`${TAG} No JSON array found in conflict detection response`);
|
||||
return fallbackStoreAll(memories);
|
||||
}
|
||||
|
||||
// Sanitize control characters inside JSON string literals that LLM may produce
|
||||
const sanitized = sanitizeJsonForParse(arrayMatch[0]);
|
||||
const parsed = JSON.parse(sanitized) as unknown[];
|
||||
|
||||
if (!Array.isArray(parsed)) {
|
||||
logger?.warn?.(`${TAG} Conflict detection response is not an array`);
|
||||
return fallbackStoreAll(memories);
|
||||
}
|
||||
|
||||
// Build decisions from LLM output
|
||||
const decisions: DedupDecision[] = [];
|
||||
const validActions = ["store", "update", "merge", "skip"];
|
||||
|
||||
for (const item of parsed) {
|
||||
if (!item || typeof item !== "object") continue;
|
||||
const d = item as Record<string, unknown>;
|
||||
|
||||
const recordId = String(d.record_id ?? "");
|
||||
// Skip entries with empty/missing record_id — they are LLM hallucinations
|
||||
if (!recordId) {
|
||||
logger?.debug?.(`${TAG} Skipping decision with empty record_id`);
|
||||
continue;
|
||||
}
|
||||
const action = String(d.action ?? "store");
|
||||
|
||||
if (!validActions.includes(action)) {
|
||||
logger?.warn?.(`${TAG} Invalid action "${action}" for record ${recordId}, defaulting to store`);
|
||||
}
|
||||
|
||||
decisions.push({
|
||||
record_id: recordId,
|
||||
action: validActions.includes(action) ? (action as DedupDecision["action"]) : "store",
|
||||
target_ids: Array.isArray(d.target_ids) ? d.target_ids.map(String) : [],
|
||||
merged_content: typeof d.merged_content === "string" ? d.merged_content : undefined,
|
||||
merged_type: VALID_TYPES.includes(d.merged_type as MemoryType) ? (d.merged_type as MemoryType) : undefined,
|
||||
merged_priority: typeof d.merged_priority === "number" ? d.merged_priority : undefined,
|
||||
merged_timestamps: Array.isArray(d.merged_timestamps) ? d.merged_timestamps.map(String) : undefined,
|
||||
});
|
||||
}
|
||||
|
||||
// Ensure all memories have a decision (fill missing with "store")
|
||||
const decidedIds = new Set(decisions.map((d) => d.record_id));
|
||||
for (const mem of memories) {
|
||||
if (!decidedIds.has(mem.record_id)) {
|
||||
logger?.debug?.(`${TAG} No decision for record ${mem.record_id}, defaulting to store`);
|
||||
decisions.push({
|
||||
record_id: mem.record_id,
|
||||
action: "store",
|
||||
target_ids: [],
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return decisions;
|
||||
} catch (err) {
|
||||
logger?.warn?.(`${TAG} Failed to parse conflict detection result: ${err instanceof Error ? err.message : String(err)}`);
|
||||
return fallbackStoreAll(memories);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fallback: store all memories when parsing fails.
|
||||
*/
|
||||
function fallbackStoreAll(memories: Array<ExtractedMemory & { record_id: string }>): DedupDecision[] {
|
||||
return memories.map((m) => ({
|
||||
record_id: m.record_id,
|
||||
action: "store" as const,
|
||||
target_ids: [],
|
||||
}));
|
||||
}
|
||||
@@ -0,0 +1,525 @@
|
||||
/**
|
||||
* L1 Memory Extractor: extracts structured memories from L0 conversation messages
|
||||
* using a single LLM call with JSON-mode structured output.
|
||||
*
|
||||
* v3: Aligned with Kenty's prompt — scene segmentation + memory extraction in one call,
|
||||
* followed by batch conflict detection.
|
||||
*
|
||||
* Pipeline:
|
||||
* 1. Read recent messages from L0 (split into background + new)
|
||||
* 2. Call LLM to extract scene-segmented memories
|
||||
* 3. Batch conflict detection against existing records
|
||||
* 4. Write to L1 JSONL files
|
||||
*/
|
||||
|
||||
import type { ConversationMessage } from "../conversation/l0-recorder.js";
|
||||
import { EXTRACT_MEMORIES_SYSTEM_PROMPT, formatExtractionPrompt } from "../prompts/l1-extraction.js";
|
||||
import { batchDedup } from "./l1-dedup.js";
|
||||
import { writeMemory, generateMemoryId } from "./l1-writer.js";
|
||||
import type { ExtractedMemory, MemoryRecord, MemoryType, DedupDecision } from "./l1-writer.js";
|
||||
import { CleanContextRunner } from "../../utils/clean-context-runner.js";
|
||||
import { sanitizeJsonForParse, shouldExtractL1 } from "../../utils/sanitize.js";
|
||||
import type { IMemoryStore } from "../store/types.js";
|
||||
import type { EmbeddingService } from "../store/embedding.js";
|
||||
import { report } from "../report/reporter.js";
|
||||
import type { LLMRunner } from "../types.js";
|
||||
|
||||
interface Logger {
|
||||
debug?: (message: string) => void;
|
||||
info: (message: string) => void;
|
||||
warn: (message: string) => void;
|
||||
error: (message: string) => void;
|
||||
}
|
||||
|
||||
const TAG = "[memory-tdai][l1-extractor]";
|
||||
|
||||
// ============================
|
||||
// Types
|
||||
// ============================
|
||||
|
||||
/** A scene segment with its extracted memories (LLM output) */
|
||||
interface SceneSegment {
|
||||
scene_name: string;
|
||||
message_ids: string[];
|
||||
memories: Array<{
|
||||
content: string;
|
||||
type: string;
|
||||
priority: number;
|
||||
source_message_ids: string[];
|
||||
metadata: Record<string, unknown>;
|
||||
}>;
|
||||
}
|
||||
|
||||
export interface L1ExtractionResult {
|
||||
/** Whether extraction succeeded */
|
||||
success: boolean;
|
||||
/** Number of memories extracted */
|
||||
extractedCount: number;
|
||||
/** Number of memories actually stored (after dedup) */
|
||||
storedCount: number;
|
||||
/** The memory records that were stored */
|
||||
records: MemoryRecord[];
|
||||
/** Scene names detected during extraction */
|
||||
sceneNames: string[];
|
||||
/** Last scene name (for continuity in next extraction) */
|
||||
lastSceneName?: string;
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Core function
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Run the full L1 extraction pipeline on conversation messages.
|
||||
*
|
||||
* @param messages - Filtered conversation messages (from L0 or directly from hook)
|
||||
* @param sessionKey - The session key
|
||||
* @param baseDir - Base data directory (~/.openclaw/memory-tdai/)
|
||||
* @param config - OpenClaw config (for LLM access)
|
||||
* @param options - Extraction options
|
||||
* @param logger - Optional logger
|
||||
*/
|
||||
export async function extractL1Memories(params: {
|
||||
messages: ConversationMessage[];
|
||||
sessionKey: string;
|
||||
sessionId?: string;
|
||||
baseDir: string;
|
||||
config: unknown;
|
||||
options?: {
|
||||
/** Max new messages to send in one extraction call */
|
||||
maxMessagesPerExtraction?: number;
|
||||
/** Max background messages for context */
|
||||
maxBackgroundMessages?: number;
|
||||
/** Enable conflict detection */
|
||||
enableDedup?: boolean;
|
||||
/** Max memories extracted per call */
|
||||
maxMemoriesPerSession?: number;
|
||||
/** LLM model override */
|
||||
model?: string;
|
||||
/** Previous scene name for continuity */
|
||||
previousSceneName?: string;
|
||||
/** Vector store for cosine similarity candidate recall */
|
||||
vectorStore?: IMemoryStore;
|
||||
/** Embedding service for computing query vectors */
|
||||
embeddingService?: EmbeddingService;
|
||||
/** Top-K candidates for conflict recall (default: 5) */
|
||||
conflictRecallTopK?: number;
|
||||
/** Override embedding timeout for capture-path calls (milliseconds) */
|
||||
embeddingTimeoutMs?: number;
|
||||
/**
|
||||
* Host-neutral LLM runner. When provided, used instead of creating
|
||||
* a CleanContextRunner (decouples from OpenClaw runtime).
|
||||
*/
|
||||
llmRunner?: LLMRunner;
|
||||
};
|
||||
logger?: Logger;
|
||||
/** Plugin instance ID for metric reporting (optional — metrics skipped if absent) */
|
||||
instanceId?: string;
|
||||
}): Promise<L1ExtractionResult> {
|
||||
const { messages, sessionKey, sessionId, baseDir, config, logger, instanceId: metricInstanceId } = params;
|
||||
const options = params.options ?? {};
|
||||
const maxNewMessages = options.maxMessagesPerExtraction ?? 10;
|
||||
const maxBgMessages = options.maxBackgroundMessages ?? 5;
|
||||
const enableDedup = options.enableDedup ?? true;
|
||||
const maxMemoriesPerSession = options.maxMemoriesPerSession ?? 10;
|
||||
|
||||
if (messages.length === 0) {
|
||||
logger?.debug?.(`${TAG} No messages to extract from`);
|
||||
return { success: true, extractedCount: 0, storedCount: 0, records: [], sceneNames: [] };
|
||||
}
|
||||
|
||||
const l1StartMs = Date.now();
|
||||
|
||||
// Quality gate: filter messages through L1 extraction rules (length, symbols,
|
||||
// prompt injection, etc.) before sending to the LLM. L0 deliberately captures
|
||||
// everything; the strict filtering happens here at L1 stage.
|
||||
const qualifiedMessages = messages.filter((m) => shouldExtractL1(m.content));
|
||||
if (qualifiedMessages.length < messages.length) {
|
||||
logger?.debug?.(
|
||||
`${TAG} L1 quality filter: ${messages.length} → ${qualifiedMessages.length} messages ` +
|
||||
`(${messages.length - qualifiedMessages.length} filtered out)`,
|
||||
);
|
||||
}
|
||||
|
||||
if (qualifiedMessages.length === 0) {
|
||||
logger?.debug?.(`${TAG} All messages filtered out by L1 quality gate`);
|
||||
return { success: true, extractedCount: 0, storedCount: 0, records: [], sceneNames: [] };
|
||||
}
|
||||
|
||||
// Split messages into background (older) + new (recent)
|
||||
const newMessages = qualifiedMessages.slice(-maxNewMessages);
|
||||
const bgEndIdx = qualifiedMessages.length - newMessages.length;
|
||||
const backgroundMessages = bgEndIdx > 0
|
||||
? qualifiedMessages.slice(Math.max(0, bgEndIdx - maxBgMessages), bgEndIdx)
|
||||
: [];
|
||||
|
||||
logger?.debug?.(`${TAG} Extracting from ${newMessages.length} new messages (+ ${backgroundMessages.length} background) [${qualifiedMessages.length} qualified from ${messages.length} input]`);
|
||||
|
||||
// Step 1: LLM extraction (scene segmentation + memory extraction)
|
||||
let scenes: SceneSegment[];
|
||||
try {
|
||||
scenes = await callLlmExtraction({
|
||||
newMessages,
|
||||
backgroundMessages,
|
||||
previousSceneName: options.previousSceneName,
|
||||
config,
|
||||
logger,
|
||||
model: options.model,
|
||||
llmRunner: options.llmRunner,
|
||||
});
|
||||
logger?.debug?.(`${TAG} LLM detected ${scenes.length} scene(s)`);
|
||||
} catch (err) {
|
||||
logger?.error(`${TAG} LLM extraction failed: ${err instanceof Error ? err.message : String(err)}`);
|
||||
return { success: false, extractedCount: 0, storedCount: 0, records: [], sceneNames: [] };
|
||||
}
|
||||
|
||||
// Flatten all memories across scenes
|
||||
const allExtracted: ExtractedMemory[] = [];
|
||||
const sceneNames: string[] = [];
|
||||
|
||||
for (const scene of scenes) {
|
||||
sceneNames.push(scene.scene_name);
|
||||
for (const mem of scene.memories) {
|
||||
const memType = normalizeType(mem.type);
|
||||
if (!memType) {
|
||||
logger?.warn?.(`${TAG} Skipping memory with invalid type "${mem.type}"`);
|
||||
continue;
|
||||
}
|
||||
allExtracted.push({
|
||||
content: mem.content,
|
||||
type: memType,
|
||||
priority: typeof mem.priority === "number" ? mem.priority : 50,
|
||||
source_message_ids: Array.isArray(mem.source_message_ids) ? mem.source_message_ids : [],
|
||||
metadata: mem.metadata ?? {},
|
||||
scene_name: scene.scene_name,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
logger?.debug?.(`${TAG} Total extracted memories: ${allExtracted.length} across ${scenes.length} scene(s)`);
|
||||
|
||||
if (allExtracted.length === 0) {
|
||||
return {
|
||||
success: true,
|
||||
extractedCount: 0,
|
||||
storedCount: 0,
|
||||
records: [],
|
||||
sceneNames,
|
||||
lastSceneName: sceneNames[sceneNames.length - 1],
|
||||
};
|
||||
}
|
||||
|
||||
// Limit per session
|
||||
let extracted = allExtracted;
|
||||
if (extracted.length > maxMemoriesPerSession) {
|
||||
logger?.debug?.(`${TAG} Limiting from ${extracted.length} to ${maxMemoriesPerSession} memories per session`);
|
||||
extracted = extracted.slice(0, maxMemoriesPerSession);
|
||||
}
|
||||
|
||||
// Assign temporary IDs to extracted memories (needed for batch dedup)
|
||||
const memoriesWithIds = extracted.map((m) => ({
|
||||
...m,
|
||||
record_id: generateMemoryId(),
|
||||
}));
|
||||
|
||||
// Step 2: Batch Conflict Detection + Write
|
||||
let storedRecords: MemoryRecord[];
|
||||
|
||||
if (enableDedup) {
|
||||
try {
|
||||
const decisions = await batchDedup({
|
||||
memories: memoriesWithIds,
|
||||
config,
|
||||
logger,
|
||||
model: options.model,
|
||||
vectorStore: options.vectorStore,
|
||||
embeddingService: options.embeddingService,
|
||||
conflictRecallTopK: options.conflictRecallTopK,
|
||||
embeddingTimeoutMs: options.embeddingTimeoutMs,
|
||||
llmRunner: options.llmRunner,
|
||||
});
|
||||
|
||||
storedRecords = await applyDecisions({
|
||||
memoriesWithIds,
|
||||
decisions,
|
||||
baseDir,
|
||||
sessionKey,
|
||||
sessionId,
|
||||
logger,
|
||||
vectorStore: options.vectorStore,
|
||||
embeddingService: options.embeddingService,
|
||||
});
|
||||
} catch (err) {
|
||||
logger?.warn?.(`${TAG} Batch dedup failed, storing all as new: ${err instanceof Error ? err.message : String(err)}`);
|
||||
storedRecords = await storeAllDirectly(memoriesWithIds, baseDir, sessionKey, sessionId, logger, options.vectorStore, options.embeddingService);
|
||||
}
|
||||
} else {
|
||||
storedRecords = await storeAllDirectly(memoriesWithIds, baseDir, sessionKey, sessionId, logger, options.vectorStore, options.embeddingService);
|
||||
}
|
||||
|
||||
logger?.info(`${TAG} Extraction complete: extracted=${extracted.length}, stored=${storedRecords.length}`);
|
||||
|
||||
// ── l1_extraction metric ──
|
||||
if (metricInstanceId && logger) {
|
||||
// Build type distribution of stored memories
|
||||
const memoriesByType: Record<string, number> = {};
|
||||
for (const r of storedRecords) {
|
||||
memoriesByType[r.type] = (memoriesByType[r.type] ?? 0) + 1;
|
||||
}
|
||||
report("l1_extraction", {
|
||||
sessionKey,
|
||||
inputMessageCount: messages.length,
|
||||
memoriesExtracted: extracted.length,
|
||||
memoriesStored: storedRecords.length,
|
||||
memoriesStoredContent: storedRecords.map((r) => ({
|
||||
content: r.content,
|
||||
type: r.type,
|
||||
scene: r.scene_name ?? null,
|
||||
})),
|
||||
memoriesByType,
|
||||
totalDurationMs: Date.now() - l1StartMs,
|
||||
success: true,
|
||||
error: null,
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
extractedCount: extracted.length,
|
||||
storedCount: storedRecords.length,
|
||||
records: storedRecords,
|
||||
sceneNames,
|
||||
lastSceneName: sceneNames[sceneNames.length - 1],
|
||||
};
|
||||
}
|
||||
|
||||
// ============================
|
||||
// LLM call
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Call LLM to extract scene-segmented memories from conversation messages.
|
||||
*/
|
||||
async function callLlmExtraction(params: {
|
||||
newMessages: ConversationMessage[];
|
||||
backgroundMessages: ConversationMessage[];
|
||||
previousSceneName?: string;
|
||||
config: unknown;
|
||||
logger?: Logger;
|
||||
model?: string;
|
||||
/** Host-neutral LLM runner — when provided, used instead of CleanContextRunner. */
|
||||
llmRunner?: LLMRunner;
|
||||
}): Promise<SceneSegment[]> {
|
||||
const { newMessages, backgroundMessages, previousSceneName, config, logger, model, llmRunner } = params;
|
||||
|
||||
const userPrompt = formatExtractionPrompt({
|
||||
newMessages,
|
||||
backgroundMessages,
|
||||
previousSceneName,
|
||||
});
|
||||
|
||||
let result: string;
|
||||
|
||||
if (llmRunner) {
|
||||
// Use the host-neutral LLMRunner interface
|
||||
result = await llmRunner.run({
|
||||
prompt: userPrompt,
|
||||
systemPrompt: EXTRACT_MEMORIES_SYSTEM_PROMPT,
|
||||
taskId: "l1-extraction",
|
||||
timeoutMs: 180_000,
|
||||
});
|
||||
} else {
|
||||
// Fallback: create CleanContextRunner (OpenClaw path)
|
||||
const runner = new CleanContextRunner({
|
||||
config,
|
||||
modelRef: model,
|
||||
enableTools: false,
|
||||
logger,
|
||||
});
|
||||
|
||||
result = await runner.run({
|
||||
prompt: userPrompt,
|
||||
systemPrompt: EXTRACT_MEMORIES_SYSTEM_PROMPT,
|
||||
taskId: "l1-extraction",
|
||||
timeoutMs: 180_000,
|
||||
});
|
||||
}
|
||||
|
||||
return parseExtractionResult(result, logger);
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse the LLM's JSON response into SceneSegment array.
|
||||
* Expected format: [{scene_name, message_ids, memories: [...]}]
|
||||
*/
|
||||
function parseExtractionResult(raw: string, logger?: Logger): SceneSegment[] {
|
||||
try {
|
||||
// Strip markdown code block wrappers if present
|
||||
let cleaned = raw.trim();
|
||||
if (cleaned.startsWith("```")) {
|
||||
cleaned = cleaned.replace(/^```(?:json)?\s*\n?/, "").replace(/\n?```\s*$/, "");
|
||||
}
|
||||
|
||||
// Try to extract JSON array
|
||||
const arrayMatch = cleaned.match(/\[[\s\S]*\]/);
|
||||
if (!arrayMatch) {
|
||||
logger?.warn?.(`${TAG} No JSON array found in extraction response`);
|
||||
return [];
|
||||
}
|
||||
|
||||
// Sanitize control characters inside JSON string literals that LLM may produce
|
||||
const sanitized = sanitizeJsonForParse(arrayMatch[0]);
|
||||
const parsed = JSON.parse(sanitized) as unknown[];
|
||||
|
||||
if (!Array.isArray(parsed)) {
|
||||
logger?.warn?.(`${TAG} Extraction response is not an array`);
|
||||
return [];
|
||||
}
|
||||
|
||||
const scenes: SceneSegment[] = [];
|
||||
for (const item of parsed) {
|
||||
if (!item || typeof item !== "object") continue;
|
||||
const s = item as Record<string, unknown>;
|
||||
|
||||
scenes.push({
|
||||
scene_name: typeof s.scene_name === "string" ? s.scene_name : "未知情境",
|
||||
message_ids: Array.isArray(s.message_ids) ? s.message_ids.map(String) : [],
|
||||
memories: Array.isArray(s.memories)
|
||||
? (s.memories as Array<Record<string, unknown>>)
|
||||
.filter((m) => m && typeof m === "object" && typeof m.content === "string" && (m.content as string).length > 0)
|
||||
.map((m) => ({
|
||||
content: String(m.content),
|
||||
type: String(m.type ?? "episodic"),
|
||||
priority: typeof m.priority === "number" ? m.priority : 50,
|
||||
source_message_ids: Array.isArray(m.source_message_ids) ? m.source_message_ids.map(String) : [],
|
||||
metadata: (m.metadata && typeof m.metadata === "object" ? m.metadata : {}) as Record<string, unknown>,
|
||||
}))
|
||||
: [],
|
||||
});
|
||||
}
|
||||
|
||||
return scenes;
|
||||
} catch (err) {
|
||||
logger?.warn?.(`${TAG} Failed to parse extraction result: ${err instanceof Error ? err.message : String(err)}`);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Write helpers
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Apply batch dedup decisions — write memories according to their decisions.
|
||||
*/
|
||||
async function applyDecisions(params: {
|
||||
memoriesWithIds: Array<ExtractedMemory & { record_id: string }>;
|
||||
decisions: DedupDecision[];
|
||||
baseDir: string;
|
||||
sessionKey: string;
|
||||
sessionId?: string;
|
||||
logger?: Logger;
|
||||
vectorStore?: IMemoryStore;
|
||||
embeddingService?: EmbeddingService;
|
||||
}): Promise<MemoryRecord[]> {
|
||||
const { memoriesWithIds, decisions, baseDir, sessionKey, sessionId, logger, vectorStore, embeddingService } = params;
|
||||
const storedRecords: MemoryRecord[] = [];
|
||||
|
||||
// Build a map from record_id → decision
|
||||
const decisionMap = new Map<string, DedupDecision>();
|
||||
for (const d of decisions) {
|
||||
decisionMap.set(d.record_id, d);
|
||||
}
|
||||
|
||||
for (const memoryWithId of memoriesWithIds) {
|
||||
const decision = decisionMap.get(memoryWithId.record_id) ?? {
|
||||
record_id: memoryWithId.record_id,
|
||||
action: "store" as const,
|
||||
target_ids: [],
|
||||
};
|
||||
|
||||
try {
|
||||
const record = await writeMemory({
|
||||
memory: memoryWithId,
|
||||
decision,
|
||||
baseDir,
|
||||
sessionKey,
|
||||
sessionId,
|
||||
logger,
|
||||
vectorStore,
|
||||
embeddingService,
|
||||
});
|
||||
|
||||
if (record) {
|
||||
storedRecords.push(record);
|
||||
}
|
||||
} catch (err) {
|
||||
logger?.warn?.(
|
||||
`${TAG} Write failed for memory "${memoryWithId.content.slice(0, 50)}...": ${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return storedRecords;
|
||||
}
|
||||
|
||||
/**
|
||||
* Store all memories directly (no dedup).
|
||||
*/
|
||||
async function storeAllDirectly(
|
||||
memoriesWithIds: Array<ExtractedMemory & { record_id: string }>,
|
||||
baseDir: string,
|
||||
sessionKey: string,
|
||||
sessionId: string | undefined,
|
||||
logger?: Logger,
|
||||
vectorStore?: IMemoryStore,
|
||||
embeddingService?: EmbeddingService,
|
||||
): Promise<MemoryRecord[]> {
|
||||
const storedRecords: MemoryRecord[] = [];
|
||||
|
||||
for (const memoryWithId of memoriesWithIds) {
|
||||
try {
|
||||
const record = await writeMemory({
|
||||
memory: memoryWithId,
|
||||
decision: {
|
||||
record_id: memoryWithId.record_id,
|
||||
action: "store",
|
||||
target_ids: [],
|
||||
},
|
||||
baseDir,
|
||||
sessionKey,
|
||||
sessionId,
|
||||
logger,
|
||||
vectorStore,
|
||||
embeddingService,
|
||||
});
|
||||
if (record) {
|
||||
storedRecords.push(record);
|
||||
}
|
||||
} catch (err) {
|
||||
logger?.warn?.(
|
||||
`${TAG} Write failed for memory "${memoryWithId.content.slice(0, 50)}...": ${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return storedRecords;
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Helpers
|
||||
// ============================
|
||||
|
||||
const VALID_TYPES: MemoryType[] = ["persona", "episodic", "instruction"];
|
||||
|
||||
function normalizeType(raw: string): MemoryType | null {
|
||||
const lower = raw.toLowerCase().trim();
|
||||
if (VALID_TYPES.includes(lower as MemoryType)) {
|
||||
return lower as MemoryType;
|
||||
}
|
||||
// Handle legacy type names
|
||||
if (lower === "episode") return "episodic";
|
||||
if (lower === "instruct") return "instruction";
|
||||
if (lower === "preference") return "persona"; // fold preference into persona
|
||||
return null;
|
||||
}
|
||||
@@ -0,0 +1,218 @@
|
||||
/**
|
||||
* L1 Memory Reader: reads persisted L1 memory records.
|
||||
*
|
||||
* Provides two data paths:
|
||||
*
|
||||
* 1. **SQLite** (preferred): `queryMemoryRecords()` — uses VectorStore's `queryL1Records()`
|
||||
* with composite indexes on (session_key, updated_time) and (session_id, updated_time)
|
||||
* for efficient session-scoped and time-range queries.
|
||||
*
|
||||
* 2. **JSONL** (fallback): `readMemoryRecords()` / `readAllMemoryRecords()` — reads from
|
||||
* `records/YYYY-MM-DD.jsonl` files. Used when VectorStore is unavailable or degraded.
|
||||
*/
|
||||
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import type { MemoryRecord, MemoryType, EpisodicMetadata } from "./l1-writer.js";
|
||||
import type { IMemoryStore, L1RecordRow, L1QueryFilter } from "../store/types.js";
|
||||
|
||||
// Re-export types that readers need
|
||||
export type { MemoryRecord, MemoryType, EpisodicMetadata } from "./l1-writer.js";
|
||||
export type { L1QueryFilter } from "../store/types.js";
|
||||
|
||||
interface Logger {
|
||||
debug?: (message: string) => void;
|
||||
info: (message: string) => void;
|
||||
warn: (message: string) => void;
|
||||
error: (message: string) => void;
|
||||
}
|
||||
|
||||
const TAG = "[memory-tdai] [l1-reader]";
|
||||
|
||||
// ============================
|
||||
// SQLite-based queries (preferred)
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Query L1 memory records from SQLite via VectorStore.
|
||||
*
|
||||
* This is the **preferred** read path — it uses the composite index
|
||||
* `idx_l1_session_updated(session_id, updated_time)` for efficient
|
||||
* session-scoped and time-range queries.
|
||||
*
|
||||
* All timestamps are UTC ISO 8601 (as stored by l1-writer's dual-write).
|
||||
*
|
||||
* Falls back to empty array if VectorStore is null or degraded.
|
||||
*/
|
||||
export async function queryMemoryRecords(
|
||||
vectorStore: IMemoryStore | null | undefined,
|
||||
filter?: L1QueryFilter,
|
||||
logger?: Logger,
|
||||
): Promise<MemoryRecord[]> {
|
||||
if (!vectorStore) {
|
||||
logger?.warn(`${TAG} queryMemoryRecords: no VectorStore available, returning empty`);
|
||||
return [];
|
||||
}
|
||||
|
||||
const rows = await vectorStore.queryL1Records(filter);
|
||||
return rows.map(rowToMemoryRecord);
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a raw SQLite L1RecordRow to a MemoryRecord (same shape as JSONL records).
|
||||
*/
|
||||
function rowToMemoryRecord(row: L1RecordRow): MemoryRecord {
|
||||
let metadata: EpisodicMetadata | Record<string, never> = {};
|
||||
try {
|
||||
metadata = JSON.parse(row.metadata_json) as EpisodicMetadata | Record<string, never>;
|
||||
} catch {
|
||||
// malformed JSON — use empty object
|
||||
}
|
||||
|
||||
// Reconstruct timestamps array from timestamp_start / timestamp_end
|
||||
const timestamps: string[] = [];
|
||||
if (row.timestamp_str) timestamps.push(row.timestamp_str);
|
||||
if (row.timestamp_start && row.timestamp_start !== row.timestamp_str) timestamps.push(row.timestamp_start);
|
||||
if (row.timestamp_end && row.timestamp_end !== row.timestamp_str && row.timestamp_end !== row.timestamp_start) {
|
||||
timestamps.push(row.timestamp_end);
|
||||
}
|
||||
|
||||
return {
|
||||
id: row.record_id,
|
||||
content: row.content,
|
||||
type: row.type as MemoryType,
|
||||
priority: row.priority,
|
||||
scene_name: row.scene_name,
|
||||
source_message_ids: [], // not stored in SQLite (vector search doesn't need them)
|
||||
metadata,
|
||||
timestamps,
|
||||
createdAt: row.created_time,
|
||||
updatedAt: row.updated_time,
|
||||
sessionKey: row.session_key,
|
||||
sessionId: row.session_id,
|
||||
};
|
||||
}
|
||||
|
||||
// ============================
|
||||
// JSONL-based reads (fallback)
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Read all memory records for a session from JSONL files.
|
||||
*
|
||||
* Current naming mode:
|
||||
* - Daily merged file: records/YYYY-MM-DD.jsonl (all sessions in one file)
|
||||
*/
|
||||
export async function readMemoryRecords(
|
||||
sessionKey: string,
|
||||
baseDir: string,
|
||||
logger?: Logger,
|
||||
): Promise<MemoryRecord[]> {
|
||||
const recordsDir = path.join(baseDir, "records");
|
||||
const dateFilePattern = /^\d{4}-\d{2}-\d{2}\.jsonl$/;
|
||||
|
||||
let entries: import("node:fs").Dirent[];
|
||||
try {
|
||||
entries = await fs.readdir(recordsDir, { withFileTypes: true });
|
||||
} catch {
|
||||
// Directory doesn't exist yet
|
||||
return [];
|
||||
}
|
||||
|
||||
const targetFiles = entries
|
||||
.filter((entry) => entry.isFile() && dateFilePattern.test(entry.name))
|
||||
.map((entry) => entry.name)
|
||||
.sort();
|
||||
|
||||
if (targetFiles.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const records: MemoryRecord[] = [];
|
||||
|
||||
for (const fileName of targetFiles) {
|
||||
const filePath = path.join(recordsDir, fileName);
|
||||
|
||||
let raw: string;
|
||||
try {
|
||||
raw = await fs.readFile(filePath, "utf-8");
|
||||
} catch {
|
||||
logger?.warn?.(`${TAG} Failed to read L1 file: ${filePath}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const lines = raw.split("\n").filter((line) => line.trim());
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const line = lines[i];
|
||||
try {
|
||||
const parsed = JSON.parse(line) as Partial<MemoryRecord>;
|
||||
if (parsed.sessionKey !== sessionKey) {
|
||||
continue;
|
||||
}
|
||||
records.push(parsed as MemoryRecord);
|
||||
} catch {
|
||||
logger?.warn?.(`${TAG} Skipping malformed JSONL line in ${filePath}:${i + 1}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
records.sort((a, b) => {
|
||||
const ta = a.updatedAt || a.createdAt || "";
|
||||
const tb = b.updatedAt || b.createdAt || "";
|
||||
return ta.localeCompare(tb);
|
||||
});
|
||||
|
||||
return records;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read ALL memory records across all session JSONL files.
|
||||
*/
|
||||
export async function readAllMemoryRecords(
|
||||
baseDir: string,
|
||||
logger?: Logger,
|
||||
): Promise<MemoryRecord[]> {
|
||||
const recordsDir = path.join(baseDir, "records");
|
||||
try {
|
||||
const files = await fs.readdir(recordsDir);
|
||||
const allRecords: MemoryRecord[] = [];
|
||||
|
||||
for (const file of files) {
|
||||
if (!file.endsWith(".jsonl")) continue;
|
||||
const filePath = path.join(recordsDir, file);
|
||||
try {
|
||||
const raw = await fs.readFile(filePath, "utf-8");
|
||||
const lines = raw.split("\n").filter((line: string) => line.trim());
|
||||
for (const line of lines) {
|
||||
try {
|
||||
allRecords.push(JSON.parse(line) as MemoryRecord);
|
||||
} catch {
|
||||
logger?.warn?.(`${TAG} Skipping malformed JSONL line in ${file}`);
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
logger?.warn?.(`${TAG} Failed to read ${file}`);
|
||||
}
|
||||
}
|
||||
|
||||
allRecords.sort((a, b) => {
|
||||
const ta = a.updatedAt || a.createdAt || "";
|
||||
const tb = b.updatedAt || b.createdAt || "";
|
||||
return ta.localeCompare(tb);
|
||||
});
|
||||
|
||||
return allRecords;
|
||||
|
||||
} catch {
|
||||
// records/ directory doesn't exist yet
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Helpers
|
||||
// ============================
|
||||
|
||||
function sanitizeFilename(name: string): string {
|
||||
return name.replace(/[<>:"/\\|?*\x00-\x1f]/g, "_");
|
||||
}
|
||||
@@ -0,0 +1,280 @@
|
||||
/**
|
||||
* L1 Memory Writer: writes extracted memories to JSONL files.
|
||||
*
|
||||
* File naming: records/YYYY-MM-DD.jsonl (daily shards, all sessions merged).
|
||||
* Each record includes sessionKey for traceability.
|
||||
*
|
||||
* Write strategy:
|
||||
* - JSONL is the append-only persistent store (source of truth for backup/recovery).
|
||||
* - VectorStore (SQLite) is the primary retrieval engine.
|
||||
* - On update/merge, old records are deleted from VectorStore in real-time;
|
||||
* JSONL is append-only and cleaned up periodically by memory-cleaner.
|
||||
*
|
||||
* Supports store (append), update, merge, and skip operations.
|
||||
*
|
||||
* v3: Aligned with Kenty's prompt output format — 3 memory types (persona/episodic/instruction),
|
||||
* numeric priority, scene_name, source_message_ids, metadata, timestamps.
|
||||
*/
|
||||
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import crypto from "node:crypto";
|
||||
import type { IMemoryStore } from "../store/types.js";
|
||||
import type { EmbeddingService } from "../store/embedding.js";
|
||||
|
||||
// ============================
|
||||
// Types
|
||||
// ============================
|
||||
|
||||
/** v3: 3 memory types aligned with Kenty's extraction prompt */
|
||||
export type MemoryType = "persona" | "episodic" | "instruction";
|
||||
|
||||
/** Metadata for episodic memories (activity time range) */
|
||||
export interface EpisodicMetadata {
|
||||
activity_start_time?: string; // ISO 8601
|
||||
activity_end_time?: string; // ISO 8601
|
||||
}
|
||||
|
||||
/**
|
||||
* A persisted memory record in L1 JSONL files.
|
||||
*
|
||||
* v3 changes from v2:
|
||||
* - `importance: "high"|"medium"|"low"` → `priority: number` (0-100, -1 for strict global instructions)
|
||||
* - Added `scene_name`, `source_message_ids`, `metadata`, `timestamps`
|
||||
* - Removed `keywords` (will be rebuilt from content for search)
|
||||
* - MemoryType reduced from 4 to 3 (removed "preference", folded into "persona")
|
||||
*/
|
||||
export interface MemoryRecord {
|
||||
/** Unique ID for dedup updates */
|
||||
id: string;
|
||||
/** Memory content */
|
||||
content: string;
|
||||
/** Memory type: persona / episodic / instruction */
|
||||
type: MemoryType;
|
||||
/** Priority score: 0-100 (higher = more important), -1 = strict global instruction */
|
||||
priority: number;
|
||||
/** Scene name this memory belongs to */
|
||||
scene_name: string;
|
||||
/** Source message IDs that contributed to this memory */
|
||||
source_message_ids: string[];
|
||||
/** Type-specific metadata (e.g., activity_start_time for episodic) */
|
||||
metadata: EpisodicMetadata | Record<string, never>;
|
||||
/** Timestamp trail: all timestamps related to this memory (for merge history tracking) */
|
||||
timestamps: string[];
|
||||
/** Creation timestamp (ISO) */
|
||||
createdAt: string;
|
||||
/** Last update timestamp (ISO) */
|
||||
updatedAt: string;
|
||||
/** Source session key (conversation channel identifier) */
|
||||
sessionKey: string;
|
||||
/** Source session ID (single conversation instance identifier) */
|
||||
sessionId: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* A memory as extracted by LLM (before dedup / persistence).
|
||||
* Matches the output format of Kenty's extraction prompt.
|
||||
*/
|
||||
export interface ExtractedMemory {
|
||||
content: string;
|
||||
type: MemoryType;
|
||||
priority: number;
|
||||
source_message_ids: string[];
|
||||
metadata: EpisodicMetadata | Record<string, never>;
|
||||
/** Scene name this memory was extracted in */
|
||||
scene_name: string;
|
||||
}
|
||||
|
||||
export type DedupAction = "store" | "update" | "merge" | "skip";
|
||||
|
||||
/**
|
||||
* v3 batch dedup decision — one per new memory, aligned with Kenty's conflict detection prompt.
|
||||
*
|
||||
* Key changes:
|
||||
* - `targetId` → `target_ids` (array, supports multi-target merge/update)
|
||||
* - Added `merged_type`, `merged_priority`, `merged_timestamps` for cross-type merge
|
||||
*/
|
||||
export interface DedupDecision {
|
||||
/** Which new memory this decision is about */
|
||||
record_id: string;
|
||||
action: DedupAction;
|
||||
/** IDs of existing records to replace/remove (for update/merge) */
|
||||
target_ids: string[];
|
||||
/** Merged/updated content text (for update/merge) */
|
||||
merged_content?: string;
|
||||
/** Best type after merge (for update/merge, may differ from original) */
|
||||
merged_type?: MemoryType;
|
||||
/** Priority after merge (for update/merge) */
|
||||
merged_priority?: number;
|
||||
/** Union of all related timestamps (for update/merge) */
|
||||
merged_timestamps?: string[];
|
||||
}
|
||||
|
||||
interface Logger {
|
||||
debug?: (message: string) => void;
|
||||
info: (message: string) => void;
|
||||
warn: (message: string) => void;
|
||||
error: (message: string) => void;
|
||||
}
|
||||
|
||||
const TAG = "[memory-tdai][l1-writer]";
|
||||
|
||||
// ============================
|
||||
// Core functions
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Generate a unique memory ID.
|
||||
*/
|
||||
export function generateMemoryId(): string {
|
||||
return `m_${Date.now()}_${crypto.randomBytes(4).toString("hex")}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Write a memory record according to the dedup decision.
|
||||
*
|
||||
* - store: append new record
|
||||
* - update: remove target records + append updated record
|
||||
* - merge: remove target records + append merged record
|
||||
* - skip: do nothing
|
||||
*
|
||||
* v3: supports multi-target removal for update/merge.
|
||||
* v3.1: optional VectorStore + EmbeddingService for dual-write (JSONL + vector).
|
||||
*/
|
||||
export async function writeMemory(params: {
|
||||
memory: ExtractedMemory;
|
||||
decision: DedupDecision;
|
||||
baseDir: string;
|
||||
sessionKey: string;
|
||||
sessionId?: string;
|
||||
logger?: Logger;
|
||||
/** Optional vector store for dual-write (JSONL + vector DB) */
|
||||
vectorStore?: IMemoryStore;
|
||||
/** Optional embedding service (required when vectorStore is provided) */
|
||||
embeddingService?: EmbeddingService;
|
||||
}): Promise<MemoryRecord | null> {
|
||||
const { memory, decision, baseDir, sessionKey, sessionId, logger, vectorStore, embeddingService } = params;
|
||||
|
||||
if (decision.action === "skip") {
|
||||
logger?.debug?.(`${TAG} Skipping memory: ${memory.content.slice(0, 50)}...`);
|
||||
return null;
|
||||
}
|
||||
|
||||
const now = new Date().toISOString();
|
||||
|
||||
// Determine final content, type, priority based on action
|
||||
let finalContent: string;
|
||||
let finalType: MemoryType;
|
||||
let finalPriority: number;
|
||||
let finalTimestamps: string[];
|
||||
|
||||
if (decision.action === "merge" || decision.action === "update") {
|
||||
finalContent = decision.merged_content ?? memory.content;
|
||||
finalType = decision.merged_type ?? memory.type;
|
||||
finalPriority = decision.merged_priority ?? memory.priority;
|
||||
finalTimestamps = decision.merged_timestamps ?? [now];
|
||||
} else {
|
||||
// store
|
||||
finalContent = memory.content;
|
||||
finalType = memory.type;
|
||||
finalPriority = memory.priority;
|
||||
finalTimestamps = [now];
|
||||
}
|
||||
|
||||
const record: MemoryRecord = {
|
||||
id: decision.record_id || generateMemoryId(),
|
||||
content: finalContent,
|
||||
type: finalType,
|
||||
priority: finalPriority,
|
||||
scene_name: memory.scene_name,
|
||||
source_message_ids: memory.source_message_ids,
|
||||
metadata: memory.metadata,
|
||||
timestamps: finalTimestamps,
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
sessionKey,
|
||||
sessionId: sessionId || "",
|
||||
};
|
||||
|
||||
const recordsDir = path.join(baseDir, "records");
|
||||
await fs.mkdir(recordsDir, { recursive: true });
|
||||
|
||||
const shardDate = formatLocalDate(new Date());
|
||||
const filePath = path.join(recordsDir, `${shardDate}.jsonl`);
|
||||
|
||||
if ((decision.action === "update" || decision.action === "merge") && decision.target_ids.length > 0) {
|
||||
// Remove target records from VectorStore (real-time deletion for retrieval accuracy).
|
||||
// JSONL is append-only — old records remain in files and are cleaned up periodically
|
||||
// by memory-cleaner (which reconciles against VectorStore as source of truth).
|
||||
if (vectorStore) {
|
||||
try {
|
||||
await vectorStore.deleteL1Batch(decision.target_ids);
|
||||
logger?.debug?.(`${TAG} VectorStore: deleted ${decision.target_ids.length} target record(s) for ${decision.action}`);
|
||||
} catch (err) {
|
||||
logger?.warn?.(
|
||||
`${TAG} VectorStore delete failed for ${decision.action}: ${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
await fs.appendFile(filePath, JSON.stringify(record) + "\n", "utf-8");
|
||||
logger?.debug?.(`${TAG} ${decision.action} memory: removed [${decision.target_ids.join(",")}] from VectorStore → ${record.id}: ${finalContent.slice(0, 80)}...`);
|
||||
} else {
|
||||
// store: append a new line
|
||||
await fs.appendFile(filePath, JSON.stringify(record) + "\n", "utf-8");
|
||||
logger?.debug?.(`${TAG} Stored memory ${record.id}: ${finalContent.slice(0, 80)}...`);
|
||||
}
|
||||
|
||||
// === Vector Store dual-write ===
|
||||
if (vectorStore) {
|
||||
try {
|
||||
logger?.debug?.(
|
||||
`${TAG} [vec-dual-write] START id=${record.id}, contentLen=${record.content.length}, ` +
|
||||
`content="${record.content.slice(0, 80)}..."`,
|
||||
);
|
||||
|
||||
let embedding: Float32Array | undefined;
|
||||
|
||||
if (embeddingService) {
|
||||
try {
|
||||
embedding = await embeddingService.embed(record.content);
|
||||
logger?.debug?.(
|
||||
`${TAG} [vec-dual-write] Embedding OK: dims=${embedding.length}, ` +
|
||||
`norm=${Math.sqrt(Array.from(embedding).reduce((s, v) => s + v * v, 0)).toFixed(4)}`,
|
||||
);
|
||||
} catch (embedErr) {
|
||||
// Embedding failed — pass undefined to upsert() which writes
|
||||
// metadata + FTS only, skipping the vec0 table.
|
||||
logger?.warn(
|
||||
`${TAG} [vec-dual-write] Embedding FAILED for id=${record.id}, ` +
|
||||
`will write metadata only: ${embedErr instanceof Error ? embedErr.message : String(embedErr)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const upsertOk = await vectorStore.upsertL1(record, embedding);
|
||||
logger?.debug?.(`${TAG} [vec-dual-write] upsert result=${upsertOk} id=${record.id}`);
|
||||
} catch (err) {
|
||||
// Vector write failure should NOT block the main JSONL write
|
||||
logger?.warn?.(
|
||||
`${TAG} [vec-dual-write] FAILED (JSONL already written) id=${record.id}: ${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
}
|
||||
} else {
|
||||
logger?.debug?.(
|
||||
`${TAG} [vec-dual-write] SKIPPED id=${record.id}: vectorStore=${!!vectorStore}`,
|
||||
);
|
||||
}
|
||||
|
||||
return record;
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Helpers
|
||||
// ============================
|
||||
|
||||
function formatLocalDate(d: Date): string {
|
||||
const y = d.getFullYear();
|
||||
const m = String(d.getMonth() + 1).padStart(2, "0");
|
||||
const day = String(d.getDate()).padStart(2, "0");
|
||||
return `${y}-${m}-${day}`;
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
|
||||
export const REPORT_CONST = {
|
||||
PLUGIN: "plugin",
|
||||
} as const;
|
||||
|
||||
export type ReportPayload = Record<string, unknown>;
|
||||
|
||||
export interface IReporter {
|
||||
reportFunc(category: string, payload: ReportPayload): void;
|
||||
}
|
||||
|
||||
// ── Singleton ──
|
||||
|
||||
let _reporter: IReporter | undefined;
|
||||
|
||||
export function initReporter(opts: {
|
||||
enabled: boolean;
|
||||
type: string;
|
||||
logger: { info: (msg: string) => void; debug?: (msg: string) => void };
|
||||
instanceId: string;
|
||||
pluginVersion: string;
|
||||
}): void {
|
||||
if (_reporter) return;
|
||||
if (!opts.enabled) return;
|
||||
switch (opts.type) {
|
||||
case "local":
|
||||
_reporter = new LocalReporter(opts.logger, opts.instanceId, opts.pluginVersion);
|
||||
break;
|
||||
// TODO: add new reporter type
|
||||
default:
|
||||
opts.logger.debug?.(`[memory-tdai] Unknown reporter type "${opts.type}", disabled reporting`);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
export function setReporter(reporter: IReporter): void {
|
||||
_reporter = reporter;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset the reporter singleton so that the next `initReporter` call takes effect.
|
||||
* Must be called at plugin re-registration (hot-reload) to pick up config changes.
|
||||
*/
|
||||
export function resetReporter(): void {
|
||||
_reporter = undefined;
|
||||
}
|
||||
|
||||
export function report(event: string, data: ReportPayload): void {
|
||||
if (!_reporter) return;
|
||||
try {
|
||||
_reporter.reportFunc(REPORT_CONST.PLUGIN, { event, ...data });
|
||||
} catch { /* never block business logic */ }
|
||||
}
|
||||
|
||||
// ── LocalReporter (default) ──
|
||||
|
||||
class LocalReporter implements IReporter {
|
||||
constructor(
|
||||
private readonly logger: { info: (msg: string) => void },
|
||||
private readonly instanceId: string,
|
||||
private readonly pluginVersion: string,
|
||||
) {}
|
||||
|
||||
reportFunc(category: string, payload: ReportPayload): void {
|
||||
try {
|
||||
this.logger.info(JSON.stringify({
|
||||
tag: "METRIC",
|
||||
category,
|
||||
plugin: "memory-tdai",
|
||||
instanceId: this.instanceId,
|
||||
pluginVersion: this.pluginVersion,
|
||||
ts: new Date().toISOString(),
|
||||
...payload,
|
||||
}));
|
||||
} catch { /* swallow */ }
|
||||
}
|
||||
}
|
||||
|
||||
// ── Instance ID (persisted per-install) ──
|
||||
|
||||
let _instanceIdCache: string | undefined;
|
||||
|
||||
export async function getOrCreateInstanceId(pluginDataDir: string): Promise<string> {
|
||||
if (_instanceIdCache) return _instanceIdCache;
|
||||
|
||||
const idFile = path.join(pluginDataDir, ".metadata", "instance_id");
|
||||
try {
|
||||
const existing = (await fs.readFile(idFile, "utf-8")).trim();
|
||||
if (existing) {
|
||||
_instanceIdCache = existing;
|
||||
return existing;
|
||||
}
|
||||
} catch { /* file doesn't exist */ }
|
||||
|
||||
const newId = randomUUID();
|
||||
await fs.mkdir(path.dirname(idFile), { recursive: true });
|
||||
await fs.writeFile(idFile, newId, "utf-8");
|
||||
_instanceIdCache = newId;
|
||||
return newId;
|
||||
}
|
||||
@@ -0,0 +1,440 @@
|
||||
/**
|
||||
* SceneExtractor: LLM-driven memory extraction into scene blocks.
|
||||
*
|
||||
* Replaces the keyword-based SceneManager.processNewMemories() with an
|
||||
* LLM agent that autonomously reads/writes scene block files using tools.
|
||||
*
|
||||
* Security: The LLM is sandboxed — workspaceDir is set to scene_blocks/
|
||||
* so it can ONLY operate on .md scene files. System files (checkpoint,
|
||||
* scene_index, persona.md) are physically invisible to the LLM.
|
||||
*
|
||||
* Flow:
|
||||
* 1. Backup + load scene index + build summaries
|
||||
* 2. Assemble extraction prompt with memories + scene context
|
||||
* 3. Run via CleanContextRunner (tools enabled, sandboxed to scene_blocks/)
|
||||
* 4. Cleanup: remove soft-deletes, sync index, update navigation
|
||||
* 5. Parse LLM text output for out-of-band persona update signals
|
||||
*/
|
||||
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { CleanContextRunner } from "../../utils/clean-context-runner.js";
|
||||
import { CheckpointManager } from "../../utils/checkpoint.js";
|
||||
import { BackupManager } from "../../utils/backup.js";
|
||||
import { readSceneIndex, syncSceneIndex } from "../scene/scene-index.js";
|
||||
import type { SceneIndexEntry } from "../scene/scene-index.js";
|
||||
import { parseSceneBlock } from "../scene/scene-format.js";
|
||||
import { generateSceneNavigation, stripSceneNavigation } from "../scene/scene-navigation.js";
|
||||
import { buildSceneExtractionPrompt } from "../prompts/scene-extraction.js";
|
||||
import { report } from "../report/reporter.js";
|
||||
import type { LLMRunner } from "../types.js";
|
||||
|
||||
const TAG = "[memory-tdai] [extractor]";
|
||||
|
||||
interface ExtractorLogger {
|
||||
debug?: (message: string) => void;
|
||||
info: (message: string) => void;
|
||||
warn: (message: string) => void;
|
||||
error: (message: string) => void;
|
||||
}
|
||||
|
||||
export interface ExtractionResult {
|
||||
memoriesProcessed: number;
|
||||
success: boolean;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export interface SceneExtractorOptions {
|
||||
dataDir: string;
|
||||
config: unknown;
|
||||
model?: string;
|
||||
maxScenes?: number;
|
||||
sceneBackupCount?: number;
|
||||
timeoutMs?: number;
|
||||
logger?: ExtractorLogger;
|
||||
/** Plugin instance ID for metric reporting (optional) */
|
||||
instanceId?: string;
|
||||
/**
|
||||
* Host-neutral LLM runner. When provided, used instead of creating
|
||||
* a CleanContextRunner (decouples from OpenClaw runtime).
|
||||
* Must be configured with `enableTools: true`.
|
||||
*/
|
||||
llmRunner?: LLMRunner;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse LLM text output for a persona update request signal.
|
||||
*
|
||||
* Supports multiple formats for robustness:
|
||||
* - Block: [PERSONA_UPDATE_REQUEST]reason: xxx[/PERSONA_UPDATE_REQUEST]
|
||||
* - Inline: PERSONA_UPDATE_REQUEST: xxx
|
||||
*/
|
||||
export function parsePersonaUpdateSignal(text: string): { reason: string } | null {
|
||||
// Block format: [PERSONA_UPDATE_REQUEST]...[/PERSONA_UPDATE_REQUEST]
|
||||
const blockMatch = text.match(
|
||||
/\[PERSONA_UPDATE_REQUEST\]\s*(?:reason:\s*)?(.+?)\s*\[\/PERSONA_UPDATE_REQUEST\]/s,
|
||||
);
|
||||
if (blockMatch) return { reason: blockMatch[1]!.trim() };
|
||||
|
||||
// Inline format: PERSONA_UPDATE_REQUEST: reason text
|
||||
const inlineMatch = text.match(
|
||||
/PERSONA_UPDATE_REQUEST:\s*(.+?)(?:\n|$)/,
|
||||
);
|
||||
if (inlineMatch) return { reason: inlineMatch[1]!.trim() };
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
export class SceneExtractor {
|
||||
private dataDir: string;
|
||||
private runner: LLMRunner;
|
||||
private maxScenes: number;
|
||||
private sceneBackupCount: number;
|
||||
private timeoutMs: number;
|
||||
private logger: ExtractorLogger | undefined;
|
||||
private instanceId: string | undefined;
|
||||
|
||||
constructor(opts: SceneExtractorOptions) {
|
||||
this.dataDir = opts.dataDir;
|
||||
this.maxScenes = opts.maxScenes ?? 15;
|
||||
this.sceneBackupCount = opts.sceneBackupCount ?? 10;
|
||||
this.timeoutMs = opts.timeoutMs ?? 300_000; // 5 min — LLM may do multiple tool calls
|
||||
this.logger = opts.logger;
|
||||
this.instanceId = opts.instanceId;
|
||||
|
||||
// Use injected LLMRunner if available, otherwise fall back to CleanContextRunner
|
||||
this.runner = opts.llmRunner ?? new CleanContextRunner({
|
||||
config: opts.config,
|
||||
modelRef: opts.model,
|
||||
enableTools: true,
|
||||
logger: opts.logger,
|
||||
});
|
||||
|
||||
this.logger?.debug?.(`${TAG} Created: dataDir=${opts.dataDir}, model=${opts.model ?? "(default)"}, maxScenes=${this.maxScenes}, timeout=${this.timeoutMs}ms`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract a batch of memories into scene blocks using the LLM agent.
|
||||
*
|
||||
* @param memories - Array of raw memory records from the API
|
||||
* @returns Extraction result with count and success flag
|
||||
*/
|
||||
async extract(memories: Array<{ content: string; created_at: string; id?: string }>): Promise<ExtractionResult> {
|
||||
const extractStartMs = Date.now();
|
||||
this.logger?.info(`${TAG} extract() start: ${memories.length} memories`);
|
||||
|
||||
if (memories.length === 0) {
|
||||
this.logger?.debug?.(`${TAG} extract() skipped: no memories`);
|
||||
return { memoriesProcessed: 0, success: true };
|
||||
}
|
||||
|
||||
const sceneBlocksDir = path.join(this.dataDir, "scene_blocks");
|
||||
const metadataDir = path.join(this.dataDir, ".metadata");
|
||||
|
||||
// Ensure directories exist
|
||||
await fs.mkdir(sceneBlocksDir, { recursive: true });
|
||||
await fs.mkdir(metadataDir, { recursive: true });
|
||||
|
||||
// Phase 1: Backup
|
||||
const backupStartMs = Date.now();
|
||||
const cpManager = new CheckpointManager(this.dataDir);
|
||||
const cp = await cpManager.read();
|
||||
const bm = new BackupManager(path.join(this.dataDir, ".backup"));
|
||||
await bm.backupDirectory(sceneBlocksDir, "scene_blocks", `offset${cp.total_processed}`, this.sceneBackupCount);
|
||||
this.logger?.debug?.(`${TAG} extract() backup phase: ${Date.now() - backupStartMs}ms`);
|
||||
|
||||
// Phase 2: Load scene index
|
||||
const indexStartMs = Date.now();
|
||||
const index = await readSceneIndex(this.dataDir);
|
||||
this.logger?.debug?.(`${TAG} extract() scene index loaded: ${index.length} entries (${Date.now() - indexStartMs}ms)`);
|
||||
|
||||
// Build scene summaries for the prompt (relative filenames only)
|
||||
const { summaries: sceneSummaries, filenames: existingSceneFiles } =
|
||||
this.buildSceneSummaries(index);
|
||||
|
||||
// Build scene count warning (tiered system)
|
||||
let sceneCountWarning: string | undefined;
|
||||
const sceneCount = index.length;
|
||||
if (sceneCount >= this.maxScenes) {
|
||||
sceneCountWarning = `当前场景数量为 **${sceneCount} 个**，已达到或超过 ${this.maxScenes} 个上限！\n**你必须先执行 MERGE 操作**，将最相似的 2-4 个场景合并为 1 个，然后再处理新记忆。\n参考合并对象：热度最低或主题高度重叠的场景。`;
|
||||
this.logger?.warn(`${TAG} extract() scene count at limit: ${sceneCount}/${this.maxScenes}`);
|
||||
} else if (sceneCount === this.maxScenes - 1) {
|
||||
sceneCountWarning = `当前场景数量为 **${sceneCount} 个**，距离上限只差 1 个！\n本次处理**只能 UPDATE 现有场景，不能 CREATE 新场景**。`;
|
||||
this.logger?.warn(`${TAG} extract() scene count near limit (CREATE blocked): ${sceneCount}/${this.maxScenes}`);
|
||||
} else if (sceneCount >= this.maxScenes - 3) {
|
||||
sceneCountWarning = `当前场景数量为 **${sceneCount} 个**，建议优先考虑 UPDATE 或主动 MERGE 相似场景。`;
|
||||
this.logger?.debug?.(`${TAG} extract() scene count approaching limit: ${sceneCount}/${this.maxScenes}`);
|
||||
}
|
||||
|
||||
// Snapshot scene index + content before LLM — used later to diff created/updated/deleted
|
||||
const preExtractIndex = new Map(index.map((e) => [e.filename, e.summary]));
|
||||
// Also snapshot scene content so we can detect content-only changes vs metadata-only changes
|
||||
const preExtractContent = new Map<string, string>();
|
||||
for (const e of index) {
|
||||
try {
|
||||
const raw = await fs.readFile(path.join(sceneBlocksDir, e.filename), "utf-8");
|
||||
const block = parseSceneBlock(raw, e.filename);
|
||||
preExtractContent.set(e.filename, block.content);
|
||||
} catch { /* non-fatal */ }
|
||||
}
|
||||
|
||||
// Phase 3: Build prompt
|
||||
const promptStartMs = Date.now();
|
||||
const memoriesJson = JSON.stringify(
|
||||
memories.map((m) => ({
|
||||
content: m.content,
|
||||
created_at: m.created_at,
|
||||
id: m.id ?? "",
|
||||
})),
|
||||
null,
|
||||
2,
|
||||
);
|
||||
|
||||
const currentTimestamp = formatTimestamp(new Date());
|
||||
|
||||
const { systemPrompt, userPrompt } = buildSceneExtractionPrompt({
|
||||
memoriesJson,
|
||||
sceneSummaries: sceneSummaries || "(无已有场景)",
|
||||
currentTimestamp,
|
||||
sceneCountWarning,
|
||||
existingSceneFiles,
|
||||
maxScenes: this.maxScenes,
|
||||
});
|
||||
this.logger?.debug?.(`${TAG} extract() prompt built: ${userPrompt.length} chars (${Date.now() - promptStartMs}ms)`);
|
||||
|
||||
// Phase 4: Run LLM agent (sandboxed to scene_blocks/)
|
||||
let llmOutput = "";
|
||||
let llmDurationMs = 0;
|
||||
try {
|
||||
this.logger?.debug?.(`${TAG} extract() starting LLM runner (timeout=${this.timeoutMs}ms, maxTokens=model default)...`);
|
||||
const runnerStartMs = Date.now();
|
||||
llmOutput = await this.runner.run({
|
||||
systemPrompt,
|
||||
prompt: userPrompt,
|
||||
taskId: `scene-extract-${Date.now()}`,
|
||||
timeoutMs: this.timeoutMs,
|
||||
// maxTokens omitted → core uses the resolved model's maxTokens from catalog
|
||||
workspaceDir: sceneBlocksDir,
|
||||
}) ?? "";
|
||||
llmDurationMs = Date.now() - runnerStartMs;
|
||||
this.logger?.debug?.(`${TAG} extract() LLM runner completed: ${llmDurationMs}ms`);
|
||||
} catch (err) {
|
||||
const errMsg = err instanceof Error ? err.message : String(err);
|
||||
const totalMs = Date.now() - extractStartMs;
|
||||
this.logger?.error(`${TAG} extract() LLM runner failed after ${totalMs}ms: ${errMsg}`);
|
||||
return { memoriesProcessed: 0, success: false, error: errMsg };
|
||||
}
|
||||
|
||||
// Phase 5: Subsequent processing — safe cleanup of soft-deleted files
|
||||
//
|
||||
// Security: The LLM has no `exec` tool and cannot run shell commands.
|
||||
// Instead, it "deletes" files by writing the marker `[DELETED]` to the file
|
||||
// (writing empty/whitespace-only content is rejected by core's write tool
|
||||
// parameter validation). Here we detect and remove those soft-deleted files
|
||||
// before syncing the index, so syncSceneIndex won't re-index stale entries.
|
||||
//
|
||||
// We also detect "META-only" files — files that contain only a META header
|
||||
// (e.g. [ARCHIVE] or [CONSOLIDATED] markers) but no actual scene content.
|
||||
// These are artifacts of LLM merges that didn't properly delete old files.
|
||||
const cleanupStartMs = Date.now();
|
||||
let cleanedCount = 0;
|
||||
try {
|
||||
const allFiles = (await fs.readdir(sceneBlocksDir)).filter((f) => f.endsWith(".md"));
|
||||
for (const file of allFiles) {
|
||||
const filePath = path.join(sceneBlocksDir, file);
|
||||
const raw = await fs.readFile(filePath, "utf-8");
|
||||
if (raw.trim().length === 0 || raw.trim() === "[DELETED]") {
|
||||
// Empty file or [DELETED] marker — soft-delete
|
||||
await fs.unlink(filePath);
|
||||
cleanedCount++;
|
||||
this.logger?.debug?.(`${TAG} extract() removed soft-deleted file: ${file}`);
|
||||
} else {
|
||||
// Check if file has only META header but no actual content
|
||||
const block = parseSceneBlock(raw, file);
|
||||
if (!block.content || block.content.trim().length === 0) {
|
||||
await fs.unlink(filePath);
|
||||
cleanedCount++;
|
||||
this.logger?.debug?.(`${TAG} extract() removed META-only file (no content): ${file}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (cleanupErr) {
|
||||
// Non-fatal — log and continue to index sync
|
||||
this.logger?.warn(`${TAG} extract() soft-delete cleanup error: ${cleanupErr instanceof Error ? cleanupErr.message : String(cleanupErr)}`);
|
||||
}
|
||||
this.logger?.debug?.(`${TAG} extract() soft-delete cleanup: removed ${cleanedCount} empty files (${Date.now() - cleanupStartMs}ms)`);
|
||||
|
||||
// Phase 6: Sync scene index (rebuilds from remaining non-empty files)
|
||||
const syncStartMs = Date.now();
|
||||
await syncSceneIndex(this.dataDir);
|
||||
this.logger?.debug?.(`${TAG} extract() scene index synced: ${Date.now() - syncStartMs}ms`);
|
||||
|
||||
// Phase 7: Update persona.md navigation (GAP-4 fix)
|
||||
const navStartMs = Date.now();
|
||||
try {
|
||||
await this.updateSceneNavigation();
|
||||
this.logger?.debug?.(`${TAG} extract() persona.md navigation updated: ${Date.now() - navStartMs}ms`);
|
||||
} catch (navErr) {
|
||||
// Non-fatal — log and continue
|
||||
this.logger?.warn(`${TAG} extract() failed to update persona navigation: ${navErr instanceof Error ? navErr.message : String(navErr)}`);
|
||||
}
|
||||
|
||||
// Phase 8: Parse LLM output for out-of-band persona update signal
|
||||
if (llmOutput) {
|
||||
const signal = parsePersonaUpdateSignal(llmOutput);
|
||||
if (signal) {
|
||||
await cpManager.setPersonaUpdateRequest(signal.reason);
|
||||
this.logger?.debug?.(`${TAG} extract() persona update requested by LLM: ${signal.reason}`);
|
||||
}
|
||||
}
|
||||
|
||||
const totalMs = Date.now() - extractStartMs;
|
||||
this.logger?.info(`${TAG} extract() completed: ${memories.length} memories processed in ${totalMs}ms`);
|
||||
|
||||
// ── l2_extraction metric ──
|
||||
if (this.instanceId && this.logger) {
|
||||
// Read updated scene index to report final state + diff against pre-extract snapshot
|
||||
let resultScenes: Array<{ title: string; summary: string; content: string; status: "created" | "updated" }> = [];
|
||||
let scenesCreated = 0;
|
||||
let scenesUpdated = 0;
|
||||
let scenesDeleted = 0;
|
||||
try {
|
||||
const finalIndex = await readSceneIndex(this.dataDir);
|
||||
const postFilenames = new Set<string>();
|
||||
for (const e of finalIndex) {
|
||||
postFilenames.add(e.filename);
|
||||
const oldSummary = preExtractIndex.get(e.filename);
|
||||
// Read scene block content from disk
|
||||
let content = "";
|
||||
try {
|
||||
const blockPath = path.join(sceneBlocksDir, e.filename);
|
||||
const raw = await fs.readFile(blockPath, "utf-8");
|
||||
const block = parseSceneBlock(raw, e.filename);
|
||||
content = block.content;
|
||||
} catch { /* file read failure is non-fatal */ }
|
||||
|
||||
if (oldSummary === undefined) {
|
||||
// New scene
|
||||
scenesCreated++;
|
||||
resultScenes.push({
|
||||
title: e.filename.replace(/\.md$/, ""),
|
||||
summary: e.summary,
|
||||
content,
|
||||
status: "created",
|
||||
});
|
||||
} else {
|
||||
// Existing scene — check if content actually changed (not just metadata)
|
||||
const oldContent = preExtractContent.get(e.filename) ?? "";
|
||||
if (content !== oldContent) {
|
||||
scenesUpdated++;
|
||||
resultScenes.push({
|
||||
title: e.filename.replace(/\.md$/, ""),
|
||||
summary: e.summary,
|
||||
content,
|
||||
status: "updated",
|
||||
});
|
||||
}
|
||||
// If only metadata (summary/heat) changed but content is the same, skip
|
||||
}
|
||||
}
|
||||
// Scenes in pre-extract but missing from post-extract = deleted
|
||||
for (const [filename] of preExtractIndex) {
|
||||
if (!postFilenames.has(filename)) {
|
||||
scenesDeleted++;
|
||||
}
|
||||
}
|
||||
} catch { /* non-fatal */ }
|
||||
|
||||
report("l2_extraction", {
|
||||
inputMemoryCount: memories.length,
|
||||
resultSceneCount: resultScenes.length,
|
||||
resultScenes,
|
||||
scenesCreated,
|
||||
scenesUpdated,
|
||||
scenesDeleted,
|
||||
llmDurationMs,
|
||||
totalDurationMs: totalMs,
|
||||
success: true,
|
||||
error: null,
|
||||
});
|
||||
}
|
||||
|
||||
return { memoriesProcessed: memories.length, success: true };
|
||||
}
|
||||
|
||||
/**
|
||||
* Build human-readable scene summaries for the prompt,
|
||||
* and collect the list of existing scene filenames (relative).
|
||||
*
|
||||
* Includes a capacity counter at the top (e.g. "当前场景总数：5 / 15")
|
||||
* so the LLM can immediately see how close it is to the limit.
|
||||
*/
|
||||
private buildSceneSummaries(
|
||||
index: SceneIndexEntry[],
|
||||
): { summaries: string; filenames: string[] } {
|
||||
if (index.length === 0) return { summaries: "", filenames: [] };
|
||||
|
||||
const lines: string[] = [];
|
||||
const filenames: string[] = [];
|
||||
|
||||
// Inject capacity counter at the top — LLM sees this first
|
||||
lines.push(`**当前场景总数：${index.length} / ${this.maxScenes}**`);
|
||||
lines.push("");
|
||||
|
||||
for (const entry of index) {
|
||||
filenames.push(entry.filename);
|
||||
lines.push(`### ${entry.filename}`);
|
||||
lines.push(`**热度**: ${entry.heat} | **更新**: ${entry.updated}`);
|
||||
lines.push(`**summary**: ${entry.summary}`);
|
||||
lines.push("");
|
||||
}
|
||||
return { summaries: lines.join("\n"), filenames };
|
||||
}
|
||||
|
||||
/**
|
||||
* Update the scene navigation section at the end of persona.md.
|
||||
*
|
||||
* Reads the current scene index, generates the navigation block, then
|
||||
* strips any existing navigation from persona.md and appends the new one.
|
||||
*
|
||||
* IMPORTANT: If the persona body is empty (PersonaGenerator hasn't run yet),
|
||||
* we skip writing to avoid creating a persona.md that only contains the
|
||||
* scene navigation. PersonaGenerator.generate() will write the full
|
||||
* persona + navigation when it runs.
|
||||
*/
|
||||
private async updateSceneNavigation(): Promise<void> {
|
||||
const personaPath = path.join(this.dataDir, "persona.md");
|
||||
const index = await readSceneIndex(this.dataDir);
|
||||
const nav = generateSceneNavigation(index);
|
||||
|
||||
let existing = "";
|
||||
try {
|
||||
existing = await fs.readFile(personaPath, "utf-8");
|
||||
} catch {
|
||||
// No persona file yet — PersonaGenerator will create it with navigation.
|
||||
// Don't write a navigation-only file.
|
||||
this.logger?.debug?.(`${TAG} updateSceneNavigation() skipped: no persona file yet, waiting for PersonaGenerator`);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!existing.trim() && !nav) return;
|
||||
|
||||
const stripped = stripSceneNavigation(existing).trimEnd();
|
||||
|
||||
// If the persona body is empty (only navigation existed), don't overwrite
|
||||
// with a navigation-only file. Let PersonaGenerator handle full generation.
|
||||
if (!stripped) {
|
||||
this.logger?.debug?.(`${TAG} updateSceneNavigation() skipped: persona body is empty, waiting for PersonaGenerator`);
|
||||
return;
|
||||
}
|
||||
|
||||
const updated = nav ? `${stripped}\n\n${nav}\n` : `${stripped}\n`;
|
||||
|
||||
// persona.md is at dataDir root, no subdir needed
|
||||
await fs.writeFile(personaPath, updated, "utf-8");
|
||||
}
|
||||
}
|
||||
|
||||
function formatTimestamp(d: Date): string {
|
||||
return d.toISOString();
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
/**
|
||||
* Scene Block file format: parse and format the META-delimited Markdown files.
|
||||
*/
|
||||
|
||||
export interface SceneBlockMeta {
|
||||
created: string;
|
||||
updated: string;
|
||||
summary: string;
|
||||
heat: number;
|
||||
}
|
||||
|
||||
export interface SceneBlock {
|
||||
filename: string;
|
||||
meta: SceneBlockMeta;
|
||||
content: string;
|
||||
}
|
||||
|
||||
const META_START = "-----META-START-----";
|
||||
const META_END = "-----META-END-----";
|
||||
|
||||
/**
|
||||
* Parse a Scene Block file into structured data.
|
||||
*/
|
||||
export function parseSceneBlock(raw: string, filename: string): SceneBlock {
|
||||
const startIdx = raw.indexOf(META_START);
|
||||
const endIdx = raw.indexOf(META_END);
|
||||
|
||||
if (startIdx === -1 || endIdx === -1) {
|
||||
// No META section — treat entire file as content
|
||||
return {
|
||||
filename,
|
||||
meta: { created: "", updated: "", summary: "", heat: 0 },
|
||||
content: raw.trim(),
|
||||
};
|
||||
}
|
||||
|
||||
const metaBlock = raw.slice(startIdx + META_START.length, endIdx).trim();
|
||||
const content = raw.slice(endIdx + META_END.length).trim();
|
||||
|
||||
const meta: SceneBlockMeta = {
|
||||
created: extractMetaField(metaBlock, "created"),
|
||||
updated: extractMetaField(metaBlock, "updated"),
|
||||
summary: extractMetaField(metaBlock, "summary"),
|
||||
heat: parseInt(extractMetaField(metaBlock, "heat"), 10) || 0,
|
||||
};
|
||||
|
||||
return { filename, meta, content };
|
||||
}
|
||||
|
||||
/**
|
||||
* Format a Scene Block back into file content.
|
||||
*/
|
||||
export function formatSceneBlock(meta: SceneBlockMeta, content: string): string {
|
||||
return `${formatMeta(meta)}\n\n${content}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Format the META section.
|
||||
*/
|
||||
export function formatMeta(meta: SceneBlockMeta): string {
|
||||
return [
|
||||
META_START,
|
||||
`created: ${meta.created}`,
|
||||
`updated: ${meta.updated}`,
|
||||
`summary: ${meta.summary}`,
|
||||
`heat: ${meta.heat}`,
|
||||
META_END,
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function extractMetaField(metaBlock: string, field: string): string {
|
||||
const re = new RegExp(`^${field}:\\s*(.*)$`, "m");
|
||||
const m = metaBlock.match(re);
|
||||
return m ? m[1]!.trim() : "";
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
/**
|
||||
* Scene Index: maintains a JSON index of all scene blocks for quick lookup.
|
||||
*/
|
||||
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { parseSceneBlock } from "./scene-format.js";
|
||||
|
||||
export interface SceneIndexEntry {
|
||||
filename: string;
|
||||
summary: string;
|
||||
heat: number;
|
||||
created: string;
|
||||
updated: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the scene index from disk.
|
||||
*
|
||||
* The index is written exclusively by syncSceneIndex() (engineering side).
|
||||
* The LLM is sandboxed to scene_blocks/ and cannot access this file.
|
||||
*/
|
||||
export async function readSceneIndex(dataDir: string): Promise<SceneIndexEntry[]> {
|
||||
const indexPath = path.join(dataDir, ".metadata", "scene_index.json");
|
||||
try {
|
||||
const raw = await fs.readFile(indexPath, "utf-8");
|
||||
const parsed = JSON.parse(raw) as Array<Record<string, unknown>>;
|
||||
if (!Array.isArray(parsed)) return [];
|
||||
|
||||
const entries: SceneIndexEntry[] = [];
|
||||
for (const item of parsed) {
|
||||
if (!item || typeof item !== "object") continue;
|
||||
|
||||
const filename = typeof item.filename === "string" ? item.filename : "";
|
||||
if (!filename) continue;
|
||||
|
||||
entries.push({
|
||||
filename,
|
||||
summary: typeof item.summary === "string" ? item.summary : "",
|
||||
heat: typeof item.heat === "number" ? item.heat : 0,
|
||||
created: typeof item.created === "string" ? item.created : "",
|
||||
updated: typeof item.updated === "string" ? item.updated : "",
|
||||
});
|
||||
}
|
||||
return entries;
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Write the scene index to disk.
|
||||
*/
|
||||
export async function writeSceneIndex(
|
||||
dataDir: string,
|
||||
entries: SceneIndexEntry[],
|
||||
): Promise<void> {
|
||||
const indexPath = path.join(dataDir, ".metadata", "scene_index.json");
|
||||
await fs.mkdir(path.dirname(indexPath), { recursive: true });
|
||||
await fs.writeFile(indexPath, JSON.stringify(entries, null, 2), "utf-8");
|
||||
}
|
||||
|
||||
/**
|
||||
* Rebuild scene index by scanning all .md files in the scene_blocks directory.
|
||||
*/
|
||||
export async function syncSceneIndex(dataDir: string): Promise<SceneIndexEntry[]> {
|
||||
const blocksDir = path.join(dataDir, "scene_blocks");
|
||||
let files: string[];
|
||||
try {
|
||||
files = (await fs.readdir(blocksDir)).filter((f) => f.endsWith(".md"));
|
||||
} catch {
|
||||
files = [];
|
||||
}
|
||||
|
||||
const entries: SceneIndexEntry[] = [];
|
||||
for (const file of files) {
|
||||
try {
|
||||
const raw = await fs.readFile(path.join(blocksDir, file), "utf-8");
|
||||
const block = parseSceneBlock(raw, file);
|
||||
entries.push({
|
||||
filename: file,
|
||||
summary: block.meta.summary,
|
||||
heat: block.meta.heat,
|
||||
created: block.meta.created,
|
||||
updated: block.meta.updated,
|
||||
});
|
||||
} catch {
|
||||
// File may have been deleted between readdir and readFile (e.g. by concurrent
|
||||
// SceneExtractor soft-delete). Skip it and continue syncing the rest.
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
await writeSceneIndex(dataDir, entries);
|
||||
return entries;
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
/**
|
||||
* Scene navigation: generates a summary navigation section appended to persona.md.
|
||||
*
|
||||
* The navigation includes **absolute** file paths so the agent can directly
|
||||
* use read_file for on-demand scene loading (progressive disclosure).
|
||||
*/
|
||||
|
||||
import path from "node:path";
|
||||
import type { SceneIndexEntry } from "./scene-index.js";
|
||||
|
||||
const NAV_HEADER = "---\n## 🗺️ Scene Navigation (Scene Index)";
|
||||
|
||||
const NAV_FOOTER = `📌 使用说明：
|
||||
- Path 是 scene block 的绝对路径，可直接使用 read_file 读取完整内容
|
||||
- 热度：该场景被记忆命中的累计次数，越高越重要
|
||||
- Summary：场景的核心要点摘要`;
|
||||
|
||||
/**
|
||||
* Build a fire-emoji string based on heat value (visual priority cue for the agent).
|
||||
*/
|
||||
function heatEmoji(heat: number): string {
|
||||
if (heat >= 1000) return " 🔥🔥🔥🔥🔥";
|
||||
if (heat >= 500) return " 🔥🔥🔥🔥";
|
||||
if (heat >= 200) return " 🔥🔥🔥";
|
||||
if (heat >= 100) return " 🔥🔥";
|
||||
if (heat >= 50) return " 🔥";
|
||||
return "";
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate the scene navigation Markdown section.
|
||||
*
|
||||
* @param entries - Scene index entries
|
||||
* @param dataDir - Absolute path to the plugin data directory; when provided,
|
||||
* scene paths are rendered as absolute paths so the agent can
|
||||
* call read_file directly without path concatenation.
|
||||
*/
|
||||
export function generateSceneNavigation(entries: SceneIndexEntry[], dataDir?: string): string {
|
||||
if (entries.length === 0) return "";
|
||||
|
||||
const sorted = [...entries].sort((a, b) => b.heat - a.heat);
|
||||
|
||||
const blocks = sorted.map((e) => {
|
||||
const scenePath = dataDir
|
||||
? path.join(dataDir, "scene_blocks", e.filename)
|
||||
: `scene_blocks/${e.filename}`;
|
||||
const pathLine = `### Path: ${scenePath}`;
|
||||
const heatLine = `**热度**: ${e.heat}${heatEmoji(e.heat)}${e.updated ? ` | **更新**: ${e.updated}` : ""}`;
|
||||
const summaryLine = `Summary: ${e.summary}`;
|
||||
return `${pathLine}\n${heatLine}\n${summaryLine}`;
|
||||
});
|
||||
|
||||
return `${NAV_HEADER}\n*以下是当前场景记忆的索引，可根据需要 read_file 读取详细内容。*\n\n${blocks.join("\n\n")}\n\n${NAV_FOOTER}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Strip the scene navigation section from persona content.
|
||||
*/
|
||||
export function stripSceneNavigation(personaContent: string): string {
|
||||
const idx = personaContent.indexOf(NAV_HEADER);
|
||||
if (idx === -1) return personaContent;
|
||||
return personaContent.slice(0, idx).trimEnd();
|
||||
}
|
||||
@@ -0,0 +1,492 @@
|
||||
/**
|
||||
* Input loading, validation, normalization, and timestamp handling for the `seed` command.
|
||||
*
|
||||
* Responsibilities:
|
||||
* 1. Load raw JSON from file
|
||||
* 2. Detect Format A (`{ sessions: [...] }`) vs Format B (`[...]`)
|
||||
* 3. Six-layer validation (file → top-level → session → round → message → timestamp consistency)
|
||||
* 4. Normalize into NormalizedInput with auto-generated sessionIds
|
||||
* 5. Timestamp all-or-none check + fill strategy
|
||||
*/
|
||||
|
||||
import fs from "node:fs";
|
||||
import crypto from "node:crypto";
|
||||
import type {
|
||||
RawSession,
|
||||
FormatA,
|
||||
ValidationError,
|
||||
NormalizedInput,
|
||||
NormalizedSession,
|
||||
NormalizedRound,
|
||||
NormalizedMessage,
|
||||
SeedCommandOptions,
|
||||
} from "./types.js";
|
||||
|
||||
// ============================
|
||||
// Public API
|
||||
// ============================
|
||||
|
||||
export interface LoadAndValidateResult {
|
||||
/** Normalized input ready for pipeline consumption. */
|
||||
input: NormalizedInput;
|
||||
/** Whether the user needs to confirm timestamp auto-fill. */
|
||||
needsTimestampConfirmation: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Load, validate, and normalize seed input from a file.
|
||||
*
|
||||
* Throws on fatal validation errors with a human-readable message
|
||||
* that includes all collected errors.
|
||||
*/
|
||||
export function loadAndValidateInput(
|
||||
opts: Pick<SeedCommandOptions, "input" | "sessionKey" | "strictRoundRole">,
|
||||
): LoadAndValidateResult {
|
||||
// Layer 1: File — read + parse
|
||||
const raw = loadRawInput(opts.input);
|
||||
|
||||
// Layer 2: Top-level — detect A vs B
|
||||
const sessions = extractSessions(raw);
|
||||
|
||||
// Layers 3-5: session / round / message validation
|
||||
const errors: ValidationError[] = [];
|
||||
validateSessions(sessions, opts.strictRoundRole, errors);
|
||||
|
||||
if (errors.length > 0) {
|
||||
throw new SeedValidationError(errors);
|
||||
}
|
||||
|
||||
// Layer 6: Timestamp consistency (all-have / all-missing / mixed → error)
|
||||
const tsResult = checkTimestampConsistency(sessions);
|
||||
if (tsResult.status === "mixed") {
|
||||
throw new SeedValidationError([{
|
||||
stage: "timestamp_consistency",
|
||||
message:
|
||||
"Timestamp consistency check failed: some messages have timestamps while others do not. " +
|
||||
"All messages must either have timestamps or none must have timestamps.",
|
||||
}]);
|
||||
}
|
||||
|
||||
// Normalize
|
||||
const normalized = normalizeSessions(sessions, opts.sessionKey);
|
||||
|
||||
return {
|
||||
input: {
|
||||
sessions: normalized.sessions,
|
||||
totalRounds: normalized.totalRounds,
|
||||
totalMessages: normalized.totalMessages,
|
||||
hasTimestamps: tsResult.status === "all_present",
|
||||
},
|
||||
needsTimestampConfirmation: tsResult.status === "all_missing",
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate and normalize seed input from an already-parsed JSON object.
|
||||
*
|
||||
* This is the gateway-friendly variant of `loadAndValidateInput` — it skips
|
||||
* the file-system layer (Layer 1) and accepts the raw parsed body directly.
|
||||
* Timestamps missing from all messages are auto-filled (no interactive
|
||||
* confirmation needed in HTTP context).
|
||||
*
|
||||
* Throws `SeedValidationError` on validation failures.
|
||||
*/
|
||||
export function validateAndNormalizeRaw(
|
||||
raw: unknown,
|
||||
opts?: { sessionKey?: string; strictRoundRole?: boolean; autoFillTimestamps?: boolean },
|
||||
): NormalizedInput {
|
||||
const strictRoundRole = opts?.strictRoundRole ?? false;
|
||||
const autoFillTimestamps = opts?.autoFillTimestamps ?? true;
|
||||
|
||||
// Layer 2: Top-level — detect A vs B
|
||||
const sessions = extractSessions(raw);
|
||||
|
||||
// Layers 3-5: session / round / message validation
|
||||
const errors: ValidationError[] = [];
|
||||
validateSessions(sessions, strictRoundRole, errors);
|
||||
|
||||
if (errors.length > 0) {
|
||||
throw new SeedValidationError(errors);
|
||||
}
|
||||
|
||||
// Layer 6: Timestamp consistency
|
||||
const tsResult = checkTimestampConsistency(sessions);
|
||||
if (tsResult.status === "mixed") {
|
||||
throw new SeedValidationError([{
|
||||
stage: "timestamp_consistency",
|
||||
message:
|
||||
"Timestamp consistency check failed: some messages have timestamps while others do not. " +
|
||||
"All messages must either have timestamps or none must have timestamps.",
|
||||
}]);
|
||||
}
|
||||
|
||||
// Normalize
|
||||
const normalized = normalizeSessions(sessions, opts?.sessionKey);
|
||||
|
||||
const input: NormalizedInput = {
|
||||
sessions: normalized.sessions,
|
||||
totalRounds: normalized.totalRounds,
|
||||
totalMessages: normalized.totalMessages,
|
||||
hasTimestamps: tsResult.status === "all_present",
|
||||
};
|
||||
|
||||
// Auto-fill timestamps in HTTP context (no interactive prompt)
|
||||
if (tsResult.status === "all_missing" && autoFillTimestamps) {
|
||||
fillTimestamps(input);
|
||||
}
|
||||
|
||||
return input;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fill timestamps for all messages when the input has no timestamps.
|
||||
*
|
||||
* Uses a single monotonically increasing counter across ALL sessions
|
||||
* to guarantee global timestamp ordering. This is critical when multiple
|
||||
* sessions share the same sessionKey — the L0 capture cursor (advanced
|
||||
* per-session) would filter out later sessions whose timestamps fall
|
||||
* below the cursor if ordering were not globally monotonic.
|
||||
*/
|
||||
export function fillTimestamps(input: NormalizedInput): void {
|
||||
let currentTs = Date.now();
|
||||
for (const session of input.sessions) {
|
||||
for (const round of session.rounds) {
|
||||
for (let i = 0; i < round.messages.length; i++) {
|
||||
// Small offset per message to maintain strict ordering
|
||||
round.messages[i]!.timestamp = currentTs;
|
||||
currentTs += 100;
|
||||
}
|
||||
}
|
||||
}
|
||||
input.hasTimestamps = true;
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Validation error class
|
||||
// ============================
|
||||
|
||||
export class SeedValidationError extends Error {
|
||||
public readonly errors: ValidationError[];
|
||||
|
||||
constructor(errors: ValidationError[]) {
|
||||
const summary = errors.map((e) => formatValidationError(e)).join("\n");
|
||||
super(`Seed input validation failed (${errors.length} error(s)):\n${summary}`);
|
||||
this.name = "SeedValidationError";
|
||||
this.errors = errors;
|
||||
}
|
||||
}
|
||||
|
||||
function formatValidationError(e: ValidationError): string {
|
||||
const parts: string[] = [` [${e.stage}]`];
|
||||
if (e.sourceIndex != null) parts.push(`session[${e.sourceIndex}]`);
|
||||
if (e.sessionKey) parts.push(`key="${e.sessionKey}"`);
|
||||
if (e.roundIndex != null) parts.push(`round[${e.roundIndex}]`);
|
||||
if (e.messageIndex != null) parts.push(`msg[${e.messageIndex}]`);
|
||||
parts.push(e.message);
|
||||
return parts.join(" ");
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Layer 1: File loading
|
||||
// ============================
|
||||
|
||||
function loadRawInput(filePath: string): unknown {
|
||||
if (!fs.existsSync(filePath)) {
|
||||
throw new SeedValidationError([{
|
||||
stage: "file",
|
||||
message: `Input file not found: ${filePath}`,
|
||||
}]);
|
||||
}
|
||||
|
||||
const content = fs.readFileSync(filePath, "utf-8").trim();
|
||||
if (!content) {
|
||||
throw new SeedValidationError([{
|
||||
stage: "file",
|
||||
message: "Input file is empty.",
|
||||
}]);
|
||||
}
|
||||
|
||||
try {
|
||||
return JSON.parse(content);
|
||||
} catch (err) {
|
||||
throw new SeedValidationError([{
|
||||
stage: "file",
|
||||
message: `JSON parse error: ${err instanceof Error ? err.message : String(err)}`,
|
||||
}]);
|
||||
}
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Layer 2: Top-level format detection
|
||||
// ============================
|
||||
|
||||
function extractSessions(raw: unknown): RawSession[] {
|
||||
// Format A: { sessions: [...] }
|
||||
if (
|
||||
raw != null &&
|
||||
typeof raw === "object" &&
|
||||
!Array.isArray(raw) &&
|
||||
"sessions" in raw
|
||||
) {
|
||||
const obj = raw as FormatA;
|
||||
if (!Array.isArray(obj.sessions)) {
|
||||
throw new SeedValidationError([{
|
||||
stage: "top_level",
|
||||
message: 'Format A detected but "sessions" is not an array.',
|
||||
}]);
|
||||
}
|
||||
return obj.sessions;
|
||||
}
|
||||
|
||||
// Format B: [...]
|
||||
if (Array.isArray(raw)) {
|
||||
return raw as RawSession[];
|
||||
}
|
||||
|
||||
throw new SeedValidationError([{
|
||||
stage: "top_level",
|
||||
message:
|
||||
"Unrecognized input format. Expected either:\n" +
|
||||
' Format A: { "sessions": [...] }\n' +
|
||||
" Format B: [ { sessionKey, conversations }, ... ]",
|
||||
}]);
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Layers 3-5: session / round / message validation
|
||||
// ============================
|
||||
|
||||
function validateSessions(
|
||||
sessions: RawSession[],
|
||||
strictRoundRole: boolean,
|
||||
errors: ValidationError[],
|
||||
): void {
|
||||
if (sessions.length === 0) {
|
||||
errors.push({
|
||||
stage: "session",
|
||||
message: "No sessions found in input.",
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
for (let si = 0; si < sessions.length; si++) {
|
||||
const session = sessions[si]!;
|
||||
|
||||
// Layer 3: session validation
|
||||
if (!session.sessionKey || typeof session.sessionKey !== "string" || session.sessionKey.trim() === "") {
|
||||
errors.push({
|
||||
stage: "session",
|
||||
sourceIndex: si,
|
||||
message: '"sessionKey" is required and must be a non-empty string.',
|
||||
});
|
||||
}
|
||||
|
||||
if (!Array.isArray(session.conversations)) {
|
||||
errors.push({
|
||||
stage: "session",
|
||||
sourceIndex: si,
|
||||
sessionKey: session.sessionKey,
|
||||
message: '"conversations" must be a two-dimensional array (array of rounds).',
|
||||
});
|
||||
continue; // Can't validate rounds
|
||||
}
|
||||
|
||||
// Check that conversations is a 2D array
|
||||
for (let ri = 0; ri < session.conversations.length; ri++) {
|
||||
const round = session.conversations[ri];
|
||||
|
||||
// Layer 4: round validation
|
||||
if (!Array.isArray(round)) {
|
||||
errors.push({
|
||||
stage: "round",
|
||||
sourceIndex: si,
|
||||
sessionKey: session.sessionKey,
|
||||
roundIndex: ri,
|
||||
message: "Round must be an array of messages.",
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
if (round.length === 0) {
|
||||
errors.push({
|
||||
stage: "round",
|
||||
sourceIndex: si,
|
||||
sessionKey: session.sessionKey,
|
||||
roundIndex: ri,
|
||||
message: "Round must be a non-empty array.",
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
// Strict round-role: each round must have at least one user and one assistant
|
||||
if (strictRoundRole) {
|
||||
const roles = new Set(round.map((m) => m.role));
|
||||
if (!roles.has("user")) {
|
||||
errors.push({
|
||||
stage: "round",
|
||||
sourceIndex: si,
|
||||
sessionKey: session.sessionKey,
|
||||
roundIndex: ri,
|
||||
message: '--strict-round-role: round must contain at least one "user" message.',
|
||||
});
|
||||
}
|
||||
if (!roles.has("assistant")) {
|
||||
errors.push({
|
||||
stage: "round",
|
||||
sourceIndex: si,
|
||||
sessionKey: session.sessionKey,
|
||||
roundIndex: ri,
|
||||
message: '--strict-round-role: round must contain at least one "assistant" message.',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Layer 5: message validation
|
||||
for (let mi = 0; mi < round.length; mi++) {
|
||||
const msg = round[mi]!;
|
||||
|
||||
if (!msg.role || typeof msg.role !== "string") {
|
||||
errors.push({
|
||||
stage: "message",
|
||||
sourceIndex: si,
|
||||
sessionKey: session.sessionKey,
|
||||
roundIndex: ri,
|
||||
messageIndex: mi,
|
||||
message: '"role" is required and must be a non-empty string.',
|
||||
});
|
||||
}
|
||||
|
||||
if (!msg.content || typeof msg.content !== "string" || msg.content.trim() === "") {
|
||||
errors.push({
|
||||
stage: "message",
|
||||
sourceIndex: si,
|
||||
sessionKey: session.sessionKey,
|
||||
roundIndex: ri,
|
||||
messageIndex: mi,
|
||||
message: '"content" is required and must be a non-empty string.',
|
||||
});
|
||||
}
|
||||
|
||||
if (msg.timestamp !== undefined) {
|
||||
if (typeof msg.timestamp === "number") {
|
||||
if (!Number.isInteger(msg.timestamp)) {
|
||||
errors.push({
|
||||
stage: "message",
|
||||
sourceIndex: si,
|
||||
sessionKey: session.sessionKey,
|
||||
roundIndex: ri,
|
||||
messageIndex: mi,
|
||||
message: '"timestamp" must be an integer (epoch milliseconds). Negative values are allowed for dates before 1970.',
|
||||
});
|
||||
}
|
||||
} else if (typeof msg.timestamp === "string") {
|
||||
if (Number.isNaN(new Date(msg.timestamp).getTime())) {
|
||||
errors.push({
|
||||
stage: "message",
|
||||
sourceIndex: si,
|
||||
sessionKey: session.sessionKey,
|
||||
roundIndex: ri,
|
||||
messageIndex: mi,
|
||||
message: `"timestamp" string is not a valid ISO 8601 date: "${msg.timestamp}".`,
|
||||
});
|
||||
}
|
||||
} else {
|
||||
errors.push({
|
||||
stage: "message",
|
||||
sourceIndex: si,
|
||||
sessionKey: session.sessionKey,
|
||||
roundIndex: ri,
|
||||
messageIndex: mi,
|
||||
message: '"timestamp" must be a number (epoch ms) or an ISO 8601 string.',
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Layer 6: Timestamp consistency
|
||||
// ============================
|
||||
|
||||
interface TimestampCheckResult {
|
||||
status: "all_present" | "all_missing" | "mixed";
|
||||
}
|
||||
|
||||
function checkTimestampConsistency(sessions: RawSession[]): TimestampCheckResult {
|
||||
let hasTs = false;
|
||||
let missingTs = false;
|
||||
|
||||
for (const session of sessions) {
|
||||
if (!Array.isArray(session.conversations)) continue;
|
||||
for (const round of session.conversations) {
|
||||
if (!Array.isArray(round)) continue;
|
||||
for (const msg of round) {
|
||||
if (msg.timestamp !== undefined && msg.timestamp !== null) {
|
||||
hasTs = true;
|
||||
} else {
|
||||
missingTs = true;
|
||||
}
|
||||
// Early exit on mixed
|
||||
if (hasTs && missingTs) {
|
||||
return { status: "mixed" };
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (hasTs && !missingTs) return { status: "all_present" };
|
||||
if (!hasTs && missingTs) return { status: "all_missing" };
|
||||
// No messages at all — treat as all_missing (will be caught by session validation)
|
||||
return { status: "all_missing" };
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Normalization
|
||||
// ============================
|
||||
|
||||
function normalizeSessions(
|
||||
sessions: RawSession[],
|
||||
fallbackSessionKey?: string,
|
||||
): { sessions: NormalizedSession[]; totalRounds: number; totalMessages: number } {
|
||||
const normalized: NormalizedSession[] = [];
|
||||
let totalRounds = 0;
|
||||
let totalMessages = 0;
|
||||
|
||||
for (let si = 0; si < sessions.length; si++) {
|
||||
const raw = sessions[si]!;
|
||||
|
||||
const sessionKey = raw.sessionKey || fallbackSessionKey || "seed-user";
|
||||
const sessionId = raw.sessionId || crypto.randomUUID();
|
||||
|
||||
const rounds: NormalizedRound[] = [];
|
||||
for (const rawRound of raw.conversations) {
|
||||
if (!Array.isArray(rawRound)) continue;
|
||||
|
||||
const messages: NormalizedMessage[] = rawRound.map((msg) => ({
|
||||
role: msg.role,
|
||||
content: msg.content,
|
||||
// Normalize timestamp: ISO string → epoch ms, number → pass-through, missing → 0 (filled later)
|
||||
timestamp: msg.timestamp == null
|
||||
? 0
|
||||
: typeof msg.timestamp === "string"
|
||||
? new Date(msg.timestamp).getTime()
|
||||
: msg.timestamp,
|
||||
}));
|
||||
|
||||
rounds.push({ messages });
|
||||
totalMessages += messages.length;
|
||||
}
|
||||
|
||||
totalRounds += rounds.length;
|
||||
normalized.push({
|
||||
sessionKey,
|
||||
sessionId,
|
||||
rounds,
|
||||
sourceIndex: si,
|
||||
});
|
||||
}
|
||||
|
||||
return { sessions: normalized, totalRounds, totalMessages };
|
||||
}
|
||||
@@ -0,0 +1,421 @@
|
||||
/**
|
||||
* Seed runtime: L0→L1→L2→L3 orchestration for the `seed` command.
|
||||
*
|
||||
* Uses the shared pipeline-factory for VectorStore/EmbeddingService init,
|
||||
* L1 runner, L2 runner, L3 runner, and persister wiring — keeping this
|
||||
* module focused on seed-specific concerns:
|
||||
* - Synchronous per-round L0 capture with progress reporting
|
||||
* - waitForL1Idle polling (L1 only — see FIXME below)
|
||||
* - Ctrl+C graceful shutdown
|
||||
*
|
||||
* FIXME: Currently we only wait for L1 to become idle before destroying the
|
||||
* pipeline. L2 (scene extraction) and L3 (persona generation) may still be
|
||||
* in-flight when `pipeline.destroy()` is called. This is intentional for now
|
||||
* to avoid excessively long seed runs, but means seed output may not include
|
||||
* the latest L2/L3 artifacts. Re-evaluate adding a full L1+L2+L3 idle wait
|
||||
* once pipeline-manager exposes reliable L2/L3 idle signals.
|
||||
*/
|
||||
|
||||
import path from "node:path";
|
||||
import { parseConfig } from "../../config.js";
|
||||
import type { MemoryTdaiConfig } from "../../config.js";
|
||||
import { performAutoCapture } from "../hooks/auto-capture.js";
|
||||
import { createPipeline, createL2Runner, createL3Runner } from "../../utils/pipeline-factory.js";
|
||||
import type { PipelineInstance, PipelineLogger } from "../../utils/pipeline-factory.js";
|
||||
import { readManifest, writeManifest } from "../../utils/manifest.js";
|
||||
import { StandaloneLLMRunnerFactory } from "../../adapters/standalone/llm-runner.js";
|
||||
import type { MemoryPipelineManager } from "../../utils/pipeline-manager.js";
|
||||
import type { LLMRunner } from "../types.js";
|
||||
import type {
|
||||
NormalizedInput,
|
||||
SeedProgress,
|
||||
SeedSummary,
|
||||
} from "./types.js";
|
||||
|
||||
const TAG = "[memory-tdai] [seed]";
|
||||
|
||||
// ============================
|
||||
// Seed pipeline options
|
||||
// ============================
|
||||
|
||||
export interface SeedRuntimeOptions {
|
||||
/** Directory to store all seed output (L0, checkpoint, vectors.db). */
|
||||
outputDir: string;
|
||||
/** OpenClaw config object (needed for LLM calls in L1). */
|
||||
openclawConfig: unknown;
|
||||
/** Raw plugin config (same shape as api.pluginConfig). */
|
||||
pluginConfig?: Record<string, unknown>;
|
||||
/** Original input file path (for manifest traceability). */
|
||||
inputFile?: string;
|
||||
/** Logger instance. */
|
||||
logger: PipelineLogger;
|
||||
/** Progress callback (called after each round). */
|
||||
onProgress?: (progress: SeedProgress) => void;
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Seed pipeline creation
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Create a seed pipeline using the shared factory, with L2/L3 runners
|
||||
* wired via shared factory functions (same logic as index.ts live runtime).
|
||||
*/
|
||||
async function createSeedPipeline(opts: SeedRuntimeOptions): Promise<{ pipeline: PipelineInstance; cfg: MemoryTdaiConfig }> {
|
||||
const { outputDir, openclawConfig, pluginConfig, logger } = opts;
|
||||
|
||||
// Parse config — all values come from pluginConfig (or parseConfig defaults)
|
||||
const cfg = parseConfig(pluginConfig);
|
||||
|
||||
logger.info(
|
||||
`${TAG} Creating seed pipeline: outputDir=${outputDir}, ` +
|
||||
`everyN=${cfg.pipeline.everyNConversations}, l1Idle=${cfg.pipeline.l1IdleTimeoutSeconds}s, ` +
|
||||
`l2Delay=${cfg.pipeline.l2DelayAfterL1Seconds}s, l2Min=${cfg.pipeline.l2MinIntervalSeconds}s, l2Max=${cfg.pipeline.l2MaxIntervalSeconds}s`,
|
||||
);
|
||||
|
||||
// Create standalone LLM runners if cfg.llm is configured.
|
||||
// Seed always runs outside OpenClaw, so it needs standalone runners
|
||||
// unless an explicit openclawConfig is provided (rare).
|
||||
let l1LlmRunner: LLMRunner | undefined;
|
||||
let l2l3LlmRunner: LLMRunner | undefined;
|
||||
|
||||
if (cfg.llm.enabled && cfg.llm.apiKey) {
|
||||
const runnerFactory = new StandaloneLLMRunnerFactory({
|
||||
config: {
|
||||
baseUrl: cfg.llm.baseUrl,
|
||||
apiKey: cfg.llm.apiKey,
|
||||
model: cfg.llm.model,
|
||||
maxTokens: cfg.llm.maxTokens,
|
||||
timeoutMs: cfg.llm.timeoutMs,
|
||||
},
|
||||
logger,
|
||||
});
|
||||
l1LlmRunner = runnerFactory.createRunner({ enableTools: false });
|
||||
l2l3LlmRunner = runnerFactory.createRunner({ enableTools: true });
|
||||
logger.info(`${TAG} Seed using standalone LLM: model=${cfg.llm.model}`);
|
||||
}
|
||||
|
||||
// Use shared factory for everything: store init, L1 runner, persister, destroy
|
||||
const pipeline = await createPipeline({
|
||||
pluginDataDir: outputDir,
|
||||
cfg,
|
||||
openclawConfig,
|
||||
logger,
|
||||
l1LlmRunner,
|
||||
});
|
||||
|
||||
// Wire L2 runner via shared factory (same logic as index.ts live runtime)
|
||||
pipeline.scheduler.setL2Runner(createL2Runner({
|
||||
pluginDataDir: outputDir,
|
||||
cfg,
|
||||
openclawConfig,
|
||||
vectorStore: pipeline.vectorStore,
|
||||
logger,
|
||||
llmRunner: l2l3LlmRunner,
|
||||
}));
|
||||
|
||||
// Wire L3 runner via shared factory (same logic as index.ts live runtime)
|
||||
pipeline.scheduler.setL3Runner(createL3Runner({
|
||||
pluginDataDir: outputDir,
|
||||
cfg,
|
||||
openclawConfig,
|
||||
vectorStore: pipeline.vectorStore,
|
||||
logger,
|
||||
llmRunner: l2l3LlmRunner,
|
||||
}));
|
||||
|
||||
return { pipeline, cfg };
|
||||
}
|
||||
|
||||
// ============================
|
||||
// waitForL1Idle
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Poll pipeline queue status until L1 is idle for a given session.
|
||||
* Modeled after benchmark-ingest.ts waitForPipelineIdle() but focused on L1 only.
|
||||
*/
|
||||
async function waitForL1Idle(
|
||||
scheduler: MemoryPipelineManager,
|
||||
sessionKeys: string[],
|
||||
logger: PipelineLogger,
|
||||
opts: {
|
||||
pollIntervalMs?: number;
|
||||
stableRounds?: number;
|
||||
maxWaitMs?: number;
|
||||
} = {},
|
||||
): Promise<void> {
|
||||
const pollInterval = opts.pollIntervalMs ?? 1_000;
|
||||
const stableRounds = opts.stableRounds ?? 3;
|
||||
const maxWait = opts.maxWaitMs ?? 300_000; // 5 min default
|
||||
|
||||
const startTime = Date.now();
|
||||
let consecutiveIdle = 0;
|
||||
|
||||
while (true) {
|
||||
const elapsed = Date.now() - startTime;
|
||||
if (elapsed > maxWait) {
|
||||
logger.warn(`${TAG} [waitL1] Max wait time reached (${(maxWait / 1000).toFixed(0)}s), proceeding`);
|
||||
break;
|
||||
}
|
||||
|
||||
const queues = scheduler.getQueueSizes();
|
||||
|
||||
// Check per-session: buffered messages + conversation count
|
||||
let totalBuffered = 0;
|
||||
let totalConversationCount = 0;
|
||||
for (const key of sessionKeys) {
|
||||
totalBuffered += scheduler.getBufferedMessageCount(key);
|
||||
const state = scheduler.getSessionState(key);
|
||||
if (state) {
|
||||
totalConversationCount += state.conversation_count;
|
||||
}
|
||||
}
|
||||
|
||||
const isIdle =
|
||||
queues.l1Idle &&
|
||||
totalBuffered === 0 &&
|
||||
totalConversationCount === 0;
|
||||
|
||||
if (isIdle) {
|
||||
consecutiveIdle++;
|
||||
if (consecutiveIdle >= stableRounds) {
|
||||
logger.debug?.(`${TAG} [waitL1] L1 stable for ${stableRounds} consecutive polls`);
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
consecutiveIdle = 0;
|
||||
logger.debug?.(
|
||||
`${TAG} [waitL1] Waiting: l1Queue=${queues.l1}, l1Pending=${queues.l1Pending}, l1Idle=${queues.l1Idle}, ` +
|
||||
`buffered=${totalBuffered}, convCount=${totalConversationCount}`,
|
||||
);
|
||||
}
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, pollInterval));
|
||||
}
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Main execution function
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Execute the seed pipeline: feed normalized input through L0 → L1.
|
||||
*
|
||||
* L2/L3 runners are wired but their completion is **not** awaited — see the
|
||||
* module-level FIXME. The pipeline is destroyed after L1 idle, so L2/L3 may
|
||||
* be interrupted mid-run.
|
||||
*
|
||||
* This is the core runtime called by `src/cli/commands/seed.ts` after
|
||||
* all input validation and user confirmation are complete.
|
||||
*/
|
||||
export async function executeSeed(
|
||||
input: NormalizedInput,
|
||||
opts: SeedRuntimeOptions,
|
||||
): Promise<SeedSummary> {
|
||||
const { logger, onProgress } = opts;
|
||||
const startTime = Date.now();
|
||||
|
||||
// Track interrupt signal
|
||||
let interrupted = false;
|
||||
const onSigint = () => {
|
||||
if (interrupted) {
|
||||
// Second Ctrl+C — force exit
|
||||
logger.warn(`${TAG} Force exit (second Ctrl+C)`);
|
||||
process.exit(1);
|
||||
}
|
||||
interrupted = true;
|
||||
logger.warn(`${TAG} Interrupt received, finishing current round and shutting down...`);
|
||||
};
|
||||
process.on("SIGINT", onSigint);
|
||||
|
||||
let pipeline: PipelineInstance | undefined;
|
||||
let totalL0Recorded = 0;
|
||||
let roundsProcessed = 0;
|
||||
|
||||
try {
|
||||
// Create and start pipeline (returns both the pipeline instance and the
|
||||
// seed-optimized config so we don't need to parse config again)
|
||||
const seed = await createSeedPipeline(opts);
|
||||
pipeline = seed.pipeline;
|
||||
const seedCfg = seed.cfg;
|
||||
|
||||
pipeline.scheduler.start({});
|
||||
logger.info(`${TAG} Pipeline started, processing ${input.sessions.length} session(s), ${input.totalRounds} round(s)`);
|
||||
|
||||
// Seed-specific: use 0 so the cold-start guard in captureAtomically()
|
||||
// does NOT filter out historical messages. In live mode Date.now()
|
||||
// prevents the first agent_end from dumping full session history,
|
||||
// but seed intentionally feeds all historical data.
|
||||
const captureStartTimestamp = 0;
|
||||
|
||||
// Process each session → each round
|
||||
// Key invariant: after every everyNConversations rounds we must wait for L1
|
||||
// to finish before feeding more rounds. Without this pause the for-loop
|
||||
// would dump all rounds into L0 back-to-back and L1 would only run once
|
||||
// with the full batch (defeating the "every N" batching semantics).
|
||||
const everyN = seedCfg.pipeline.everyNConversations;
|
||||
|
||||
for (const session of input.sessions) {
|
||||
if (interrupted) break;
|
||||
|
||||
logger.info(`${TAG} Session: key="${session.sessionKey}" id="${session.sessionId}" rounds=${session.rounds.length}`);
|
||||
|
||||
for (let ri = 0; ri < session.rounds.length; ri++) {
|
||||
if (interrupted) break;
|
||||
|
||||
const round = session.rounds[ri]!;
|
||||
roundsProcessed++;
|
||||
|
||||
// Build messages in the format expected by performAutoCapture.
|
||||
// Field must be named "timestamp" (not "ts") because l0-recorder's
|
||||
// extractUserAssistantMessages reads m.timestamp for incremental filtering.
|
||||
const messages = round.messages.map((m) => ({
|
||||
role: m.role,
|
||||
content: m.content,
|
||||
timestamp: m.timestamp,
|
||||
}));
|
||||
|
||||
try {
|
||||
const result = await performAutoCapture({
|
||||
messages,
|
||||
sessionKey: session.sessionKey,
|
||||
sessionId: session.sessionId,
|
||||
cfg: seedCfg,
|
||||
pluginDataDir: opts.outputDir,
|
||||
logger,
|
||||
scheduler: pipeline.scheduler,
|
||||
pluginStartTimestamp: captureStartTimestamp,
|
||||
vectorStore: pipeline.vectorStore,
|
||||
embeddingService: pipeline.embeddingService,
|
||||
});
|
||||
|
||||
totalL0Recorded += result.l0RecordedCount;
|
||||
} catch (err) {
|
||||
logger.error(
|
||||
`${TAG} L0 capture failed for session="${session.sessionKey}" round=${ri}: ` +
|
||||
`${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
}
|
||||
|
||||
// Report progress
|
||||
onProgress?.({
|
||||
currentRound: roundsProcessed,
|
||||
totalRounds: input.totalRounds,
|
||||
sessionKey: session.sessionKey,
|
||||
stage: "l0_captured",
|
||||
});
|
||||
|
||||
// After every N rounds, wait for the triggered L1 to finish before
|
||||
// feeding the next batch. This keeps L1 batches aligned with the
|
||||
// everyNConversations boundary instead of letting all rounds pile up.
|
||||
const roundInSession = ri + 1; // 1-based
|
||||
if (roundInSession % everyN === 0 && !interrupted) {
|
||||
onProgress?.({
|
||||
currentRound: roundsProcessed,
|
||||
totalRounds: input.totalRounds,
|
||||
sessionKey: session.sessionKey,
|
||||
stage: "l1_waiting",
|
||||
});
|
||||
|
||||
logger.info(
|
||||
`${TAG} Pausing after round ${roundInSession}/${session.rounds.length} ` +
|
||||
`for session="${session.sessionKey}" — waiting for L1 to drain`,
|
||||
);
|
||||
|
||||
await waitForL1Idle(
|
||||
pipeline.scheduler,
|
||||
[session.sessionKey],
|
||||
logger,
|
||||
{ pollIntervalMs: 500, stableRounds: 2, maxWaitMs: 120_000 },
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// After all rounds for this session, wait for any residual L1 work
|
||||
// (handles the tail when total rounds is not a multiple of everyN)
|
||||
if (!interrupted) {
|
||||
onProgress?.({
|
||||
currentRound: roundsProcessed,
|
||||
totalRounds: input.totalRounds,
|
||||
sessionKey: session.sessionKey,
|
||||
stage: "l1_waiting",
|
||||
});
|
||||
|
||||
await waitForL1Idle(
|
||||
pipeline.scheduler,
|
||||
[session.sessionKey],
|
||||
logger,
|
||||
{ pollIntervalMs: 1_000, stableRounds: 3, maxWaitMs: 300_000 },
|
||||
);
|
||||
|
||||
logger.info(`${TAG} L1 idle for session="${session.sessionKey}"`);
|
||||
}
|
||||
}
|
||||
|
||||
// Final wait for all sessions
|
||||
if (!interrupted) {
|
||||
const allKeys = input.sessions.map((s) => s.sessionKey);
|
||||
logger.info(`${TAG} Final L1 idle wait for all sessions...`);
|
||||
await waitForL1Idle(
|
||||
pipeline.scheduler,
|
||||
allKeys,
|
||||
logger,
|
||||
{ pollIntervalMs: 1_000, stableRounds: 3, maxWaitMs: 300_000 },
|
||||
);
|
||||
}
|
||||
} finally {
|
||||
process.removeListener("SIGINT", onSigint);
|
||||
|
||||
// Graceful shutdown
|
||||
if (pipeline) {
|
||||
try {
|
||||
await pipeline.destroy();
|
||||
} catch (err) {
|
||||
logger.error(`${TAG} Pipeline destroy error: ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const durationMs = Date.now() - startTime;
|
||||
|
||||
const summary: SeedSummary = {
|
||||
sessionsProcessed: input.sessions.length,
|
||||
roundsProcessed,
|
||||
messagesProcessed: input.totalMessages,
|
||||
l0RecordedCount: totalL0Recorded,
|
||||
durationMs,
|
||||
outputDir: opts.outputDir,
|
||||
};
|
||||
|
||||
if (interrupted) {
|
||||
logger.warn(`${TAG} Seed interrupted after ${roundsProcessed}/${input.totalRounds} rounds`);
|
||||
} else {
|
||||
logger.info(
|
||||
`${TAG} Seed complete: sessions=${summary.sessionsProcessed}, ` +
|
||||
`rounds=${summary.roundsProcessed}, messages=${summary.messagesProcessed}, ` +
|
||||
`l0Recorded=${summary.l0RecordedCount}, duration=${(durationMs / 1000).toFixed(1)}s`,
|
||||
);
|
||||
}
|
||||
|
||||
// Append seed info to manifest (non-fatal if it fails)
|
||||
try {
|
||||
const manifest = readManifest(opts.outputDir);
|
||||
if (manifest) {
|
||||
manifest.seed = {
|
||||
inputFile: opts.inputFile ? path.basename(opts.inputFile) : undefined,
|
||||
sessions: summary.sessionsProcessed,
|
||||
rounds: summary.roundsProcessed,
|
||||
messages: summary.messagesProcessed,
|
||||
startedAt: new Date(startTime).toISOString(),
|
||||
completedAt: new Date().toISOString(),
|
||||
};
|
||||
writeManifest(opts.outputDir, manifest);
|
||||
logger.info(`${TAG} Manifest updated with seed info`);
|
||||
}
|
||||
} catch (err) {
|
||||
logger.warn(`${TAG} Failed to update manifest with seed info (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
|
||||
return summary;
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
/**
|
||||
* Shared type definitions for the `seed` command.
|
||||
*
|
||||
* Covers:
|
||||
* - Raw input shapes (Format A / B / JSONL)
|
||||
* - Normalized internal structures
|
||||
* - Validation error descriptors
|
||||
*/
|
||||
|
||||
// ============================
|
||||
// Raw input types (before validation)
|
||||
// ============================
|
||||
|
||||
/** A single message in a conversation round. */
|
||||
export interface RawMessage {
|
||||
role: string;
|
||||
content: string;
|
||||
/**
|
||||
* Epoch milliseconds (number) **or** ISO 8601 string (e.g. `"2024-04-01T12:00:00Z"`).
|
||||
* ISO strings are parsed via `new Date()` during normalization and
|
||||
* stored internally as epoch ms.
|
||||
*/
|
||||
timestamp?: number | string;
|
||||
}
|
||||
|
||||
/** A single session entry (shared between Format A wrapper and Format B array). */
|
||||
export interface RawSession {
|
||||
sessionKey: string;
|
||||
sessionId?: string;
|
||||
conversations: RawMessage[][];
|
||||
}
|
||||
|
||||
/** Format A: `{ sessions: [...] }` */
|
||||
export interface FormatA {
|
||||
sessions: RawSession[];
|
||||
}
|
||||
|
||||
/** Format B: `[...]` (top-level array of sessions) */
|
||||
export type FormatB = RawSession[];
|
||||
|
||||
// ============================
|
||||
// Normalized types (after validation)
|
||||
// ============================
|
||||
|
||||
export interface NormalizedMessage {
|
||||
role: string;
|
||||
content: string;
|
||||
/** Epoch ms — always present after normalization (filled if originally missing). */
|
||||
timestamp: number;
|
||||
}
|
||||
|
||||
export interface NormalizedRound {
|
||||
messages: NormalizedMessage[];
|
||||
}
|
||||
|
||||
export interface NormalizedSession {
|
||||
sessionKey: string;
|
||||
sessionId: string;
|
||||
rounds: NormalizedRound[];
|
||||
/** Index in the original input array (for progress reporting). */
|
||||
sourceIndex: number;
|
||||
}
|
||||
|
||||
export interface NormalizedInput {
|
||||
sessions: NormalizedSession[];
|
||||
/** Total number of rounds across all sessions. */
|
||||
totalRounds: number;
|
||||
/** Total number of messages across all sessions. */
|
||||
totalMessages: number;
|
||||
/** Whether timestamps were present in the original input. */
|
||||
hasTimestamps: boolean;
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Validation
|
||||
// ============================
|
||||
|
||||
/** Stages where a validation error can occur. */
|
||||
export type ValidationStage =
|
||||
| "file"
|
||||
| "top_level"
|
||||
| "session"
|
||||
| "round"
|
||||
| "message"
|
||||
| "timestamp_consistency";
|
||||
|
||||
/** A single validation error with location context. */
|
||||
export interface ValidationError {
|
||||
stage: ValidationStage;
|
||||
sourceIndex?: number;
|
||||
sessionKey?: string;
|
||||
roundIndex?: number;
|
||||
messageIndex?: number;
|
||||
message: string;
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Seed command options (from CLI)
|
||||
// ============================
|
||||
|
||||
export interface SeedCommandOptions {
|
||||
/** Path to input file (required). */
|
||||
input: string;
|
||||
/** Output directory (optional, auto-generated if missing). */
|
||||
outputDir?: string;
|
||||
/** Fallback session key when input lacks one. */
|
||||
sessionKey?: string;
|
||||
/** Strict round-role validation (each round must have user + assistant). */
|
||||
strictRoundRole: boolean;
|
||||
/** Skip interactive confirmations. */
|
||||
yes: boolean;
|
||||
/** Path to memory-tdai config override file (JSON, deep-merged on top of current plugin config). */
|
||||
configFile?: string;
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Seed runtime types
|
||||
// ============================
|
||||
|
||||
/** Progress info emitted during seed execution. */
|
||||
export interface SeedProgress {
|
||||
/** Current round index (1-based, across all sessions). */
|
||||
currentRound: number;
|
||||
/** Total rounds. */
|
||||
totalRounds: number;
|
||||
/** Current session key. */
|
||||
sessionKey: string;
|
||||
/** Current stage description. */
|
||||
stage: string;
|
||||
}
|
||||
|
||||
/** Final summary after seed completes. */
|
||||
export interface SeedSummary {
|
||||
sessionsProcessed: number;
|
||||
roundsProcessed: number;
|
||||
messagesProcessed: number;
|
||||
l0RecordedCount: number;
|
||||
durationMs: number;
|
||||
outputDir: string;
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
/**
|
||||
* BM25 Sparse Vector Encoding Client.
|
||||
*
|
||||
* HTTP client for the BM25 Python sidecar service (bm25_server.py).
|
||||
* Used by TCVDB backend to generate sparse vectors for hybridSearch.
|
||||
*
|
||||
* Two operations:
|
||||
* - `encodeTexts(texts)` — encode documents for upsert (TF-based)
|
||||
* - `encodeQueries(texts)` — encode queries for search (IDF-based)
|
||||
*
|
||||
* Graceful degradation: if the sidecar is unreachable, all methods
|
||||
* return empty arrays and `isHealthy()` returns false. Callers can
|
||||
* check health to dynamically downgrade to pure semantic search.
|
||||
*/
|
||||
|
||||
// ============================
|
||||
// Types
|
||||
// ============================
|
||||
|
||||
/** Sparse vector: array of [token_hash, weight] pairs. */
|
||||
export type SparseVector = Array<[number, number]>;
|
||||
|
||||
interface Logger {
|
||||
debug?: (message: string) => void;
|
||||
info: (message: string) => void;
|
||||
warn: (message: string) => void;
|
||||
error: (message: string) => void;
|
||||
}
|
||||
|
||||
export interface BM25ClientConfig {
|
||||
/** Sidecar service URL (default: "http://127.0.0.1:8084") */
|
||||
serviceUrl: string;
|
||||
/** Request timeout in ms (default: 5000) */
|
||||
timeout: number;
|
||||
}
|
||||
|
||||
interface EncodeResponse {
|
||||
vectors: SparseVector[];
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Implementation
|
||||
// ============================
|
||||
|
||||
const TAG = "[memory-tdai][bm25-client]";
|
||||
|
||||
export class BM25Client {
|
||||
private readonly baseUrl: string;
|
||||
private readonly timeout: number;
|
||||
private readonly logger?: Logger;
|
||||
|
||||
/** Cached health status to avoid repeated checks on every call. */
|
||||
private _healthy: boolean | undefined;
|
||||
private _lastHealthCheck = 0;
|
||||
private static readonly HEALTH_CHECK_INTERVAL_MS = 30_000; // re-check every 30s
|
||||
|
||||
constructor(config: BM25ClientConfig, logger?: Logger) {
|
||||
this.baseUrl = config.serviceUrl.replace(/\/+$/, "");
|
||||
this.timeout = config.timeout;
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
/**
|
||||
* Encode document texts for upsert (TF-based BM25 scoring).
|
||||
* Returns one SparseVector per input text.
|
||||
* Returns empty array on error (non-throwing).
|
||||
*/
|
||||
async encodeTexts(texts: string[]): Promise<SparseVector[]> {
|
||||
if (texts.length === 0) return [];
|
||||
return this._encode("/encode_texts", texts);
|
||||
}
|
||||
|
||||
/**
|
||||
* Encode query texts for search (IDF-based BM25 scoring).
|
||||
* Returns one SparseVector per input text.
|
||||
* Returns empty array on error (non-throwing).
|
||||
*/
|
||||
async encodeQueries(texts: string[]): Promise<SparseVector[]> {
|
||||
if (texts.length === 0) return [];
|
||||
return this._encode("/encode_queries", texts);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the BM25 sidecar is reachable.
|
||||
* Result is cached for 30 seconds to avoid spamming health checks.
|
||||
*/
|
||||
async isHealthy(): Promise<boolean> {
|
||||
const now = Date.now();
|
||||
if (
|
||||
this._healthy !== undefined &&
|
||||
now - this._lastHealthCheck < BM25Client.HEALTH_CHECK_INTERVAL_MS
|
||||
) {
|
||||
return this._healthy;
|
||||
}
|
||||
|
||||
try {
|
||||
const resp = await fetch(`${this.baseUrl}/health`, {
|
||||
signal: AbortSignal.timeout(3000),
|
||||
});
|
||||
this._healthy = resp.ok;
|
||||
} catch {
|
||||
this._healthy = false;
|
||||
}
|
||||
this._lastHealthCheck = now;
|
||||
|
||||
if (!this._healthy) {
|
||||
this.logger?.warn(`${TAG} BM25 sidecar health check failed (${this.baseUrl})`);
|
||||
}
|
||||
|
||||
return this._healthy;
|
||||
}
|
||||
|
||||
// ── Internal ──────────────────────────────────────────────────
|
||||
|
||||
private async _encode(path: string, texts: string[]): Promise<SparseVector[]> {
|
||||
try {
|
||||
const resp = await fetch(`${this.baseUrl}${path}`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ texts }),
|
||||
signal: AbortSignal.timeout(this.timeout),
|
||||
});
|
||||
|
||||
if (!resp.ok) {
|
||||
const errBody = await resp.text().catch(() => "(unreadable)");
|
||||
this.logger?.warn(
|
||||
`${TAG} ${path} HTTP ${resp.status}: ${errBody.slice(0, 200)}`,
|
||||
);
|
||||
return [];
|
||||
}
|
||||
|
||||
const json = (await resp.json()) as EncodeResponse;
|
||||
return json.vectors ?? [];
|
||||
} catch (err) {
|
||||
// Mark unhealthy on connection errors
|
||||
this._healthy = false;
|
||||
this._lastHealthCheck = Date.now();
|
||||
|
||||
this.logger?.warn(
|
||||
`${TAG} ${path} failed: ${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Factory
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Create a BM25Client if BM25 is enabled in config.
|
||||
* Returns undefined if disabled — callers should check before using.
|
||||
*/
|
||||
export function createBM25Client(
|
||||
config: { enabled: boolean; serviceUrl: string; timeout: number },
|
||||
logger?: Logger,
|
||||
): BM25Client | undefined {
|
||||
if (!config.enabled) {
|
||||
logger?.info(`${TAG} BM25 sparse encoding disabled`);
|
||||
return undefined;
|
||||
}
|
||||
logger?.info(`${TAG} BM25 client → ${config.serviceUrl}`);
|
||||
return new BM25Client(
|
||||
{ serviceUrl: config.serviceUrl, timeout: config.timeout },
|
||||
logger,
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
/**
|
||||
* Local BM25 Sparse Vector Encoder.
|
||||
*
|
||||
* Pure TypeScript replacement for the Python sidecar BM25 client.
|
||||
* Uses @tencentdb-agent-memory/tcvdb-text package for tokenization (jieba-wasm) and BM25 encoding.
|
||||
*
|
||||
* Two operations (same contract as the old BM25Client):
|
||||
* - `encodeTexts(texts)` — encode documents for upsert (TF-based)
|
||||
* - `encodeQueries(texts)` — encode queries for search (IDF-based)
|
||||
*/
|
||||
|
||||
import { BM25Encoder } from "@tencentdb-agent-memory/tcvdb-text";
|
||||
import type { SparseVector } from "@tencentdb-agent-memory/tcvdb-text";
|
||||
|
||||
export type { SparseVector };
|
||||
|
||||
interface Logger {
|
||||
debug?: (message: string) => void;
|
||||
info: (message: string) => void;
|
||||
warn: (message: string) => void;
|
||||
error: (message: string) => void;
|
||||
}
|
||||
|
||||
export interface BM25LocalConfig {
|
||||
/** Whether BM25 sparse encoding is enabled (default: true) */
|
||||
enabled: boolean;
|
||||
/** Language for BM25 pre-trained params: "zh" or "en" (default: "zh") */
|
||||
language?: "zh" | "en";
|
||||
}
|
||||
|
||||
const TAG = "[memory-tdai][bm25-local]";
|
||||
|
||||
// ============================
|
||||
// Implementation
|
||||
// ============================
|
||||
|
||||
export class BM25LocalEncoder {
|
||||
private readonly encoder: BM25Encoder;
|
||||
private readonly logger?: Logger;
|
||||
|
||||
constructor(language: "zh" | "en" = "zh", logger?: Logger) {
|
||||
this.logger = logger;
|
||||
this.encoder = BM25Encoder.default(language);
|
||||
logger?.debug?.(`${TAG} Initialized BM25 local encoder (language=${language})`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Encode document texts for upsert (TF-based BM25 scoring).
|
||||
* Returns one SparseVector per input text.
|
||||
*/
|
||||
encodeTexts(texts: string[]): SparseVector[] {
|
||||
if (texts.length === 0) return [];
|
||||
try {
|
||||
return this.encoder.encodeTexts(texts);
|
||||
} catch (err) {
|
||||
this.logger?.warn(
|
||||
`${TAG} encodeTexts failed: ${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Encode query texts for search (IDF-based BM25 scoring).
|
||||
* Returns one SparseVector per input text.
|
||||
*/
|
||||
encodeQueries(texts: string[]): SparseVector[] {
|
||||
if (texts.length === 0) return [];
|
||||
try {
|
||||
return this.encoder.encodeQueries(texts);
|
||||
} catch (err) {
|
||||
this.logger?.warn(
|
||||
`${TAG} encodeQueries failed: ${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Factory
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Create a BM25LocalEncoder if BM25 is enabled in config.
|
||||
* Returns undefined if disabled — callers should check before using.
|
||||
*/
|
||||
export function createBM25Encoder(
|
||||
config: BM25LocalConfig,
|
||||
logger?: Logger,
|
||||
): BM25LocalEncoder | undefined {
|
||||
if (!config.enabled) {
|
||||
logger?.debug?.(`${TAG} BM25 sparse encoding disabled`);
|
||||
return undefined;
|
||||
}
|
||||
return new BM25LocalEncoder(config.language ?? "zh", logger);
|
||||
}
|
||||
@@ -0,0 +1,651 @@
|
||||
/**
|
||||
* Embedding Service: converts text to vector embeddings.
|
||||
*
|
||||
* Supports two providers:
|
||||
* - "openai": OpenAI-compatible embedding APIs (OpenAI, Azure OpenAI, self-hosted)
|
||||
* - "local": node-llama-cpp with embeddinggemma-300m GGUF model (fully offline)
|
||||
*
|
||||
* When no remote embedding is configured, automatically falls back to local provider.
|
||||
*
|
||||
* Design:
|
||||
* - Single `embed()` for one text, `embedBatch()` for multiple.
|
||||
* - `getDimensions()` returns configured vector dimensions.
|
||||
* - Throws on failure; callers decide fallback strategy.
|
||||
*/
|
||||
|
||||
// ============================
|
||||
// Types
|
||||
// ============================
|
||||
|
||||
export interface OpenAIEmbeddingConfig {
|
||||
/** Provider identifier — any value other than "local" (e.g. "openai", "deepseek", "azure", "qclaw") */
|
||||
provider: string;
|
||||
/** API base URL (required — must be specified by user, e.g. "https://api.openai.com/v1") */
|
||||
baseUrl: string;
|
||||
/** API Key (required) */
|
||||
apiKey: string;
|
||||
/** Model name (required — must be specified by user) */
|
||||
model: string;
|
||||
/** Output dimensions (required — must match the chosen model) */
|
||||
dimensions: number;
|
||||
/** Local proxy URL (only for provider="qclaw") — requests are forwarded through this proxy with Remote-URL header */
|
||||
proxyUrl?: string;
|
||||
/** Max input text length in characters before truncation (default: 5000). */
|
||||
maxInputChars?: number;
|
||||
/** Timeout per API call in milliseconds (default: 10000). */
|
||||
timeoutMs?: number;
|
||||
}
|
||||
|
||||
export interface LocalEmbeddingConfig {
|
||||
provider: "local";
|
||||
/** Custom GGUF model path (default: embeddinggemma-300m from HuggingFace) */
|
||||
modelPath?: string;
|
||||
/** Model cache directory (default: node-llama-cpp default cache) */
|
||||
modelCacheDir?: string;
|
||||
}
|
||||
|
||||
export type EmbeddingConfig = OpenAIEmbeddingConfig | LocalEmbeddingConfig;
|
||||
|
||||
/** Identifies the embedding provider + model for change detection. */
|
||||
export interface EmbeddingProviderInfo {
|
||||
/** Provider identifier (e.g. "local", "openai", "deepseek") */
|
||||
provider: string;
|
||||
/** Model identifier (e.g. "embeddinggemma-300m", "text-embedding-3-large") */
|
||||
model: string;
|
||||
}
|
||||
|
||||
export interface EmbeddingCallOptions {
|
||||
/** Override the default timeout for this call (milliseconds). */
|
||||
timeoutMs?: number;
|
||||
}
|
||||
|
||||
export interface EmbeddingService {
|
||||
/** Get embedding for a single text */
|
||||
embed(text: string, options?: EmbeddingCallOptions): Promise<Float32Array>;
|
||||
/** Get embeddings for multiple texts (batched API call) */
|
||||
embedBatch(texts: string[], options?: EmbeddingCallOptions): Promise<Float32Array[]>;
|
||||
/** Return the configured vector dimensions */
|
||||
getDimensions(): number;
|
||||
/** Return provider + model identifiers for change detection */
|
||||
getProviderInfo(): EmbeddingProviderInfo;
|
||||
/**
|
||||
* Whether the service is ready to serve embed requests.
|
||||
* For remote providers (OpenAI), always true (stateless HTTP).
|
||||
* For local providers, true only after model download + load completes.
|
||||
*/
|
||||
isReady(): boolean;
|
||||
/**
|
||||
* Start background warmup (model download + load).
|
||||
* For remote providers, this is a no-op.
|
||||
* For local providers, triggers async initialization without blocking.
|
||||
* Safe to call multiple times (idempotent).
|
||||
*/
|
||||
startWarmup(): void;
|
||||
/** Optional: release resources (model memory, GPU, etc.) on shutdown */
|
||||
close?(): void | Promise<void>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Error thrown when embed() / embedBatch() is called before the local
|
||||
* embedding model has finished downloading and loading.
|
||||
* Callers should catch this and fall back to keyword-only mode.
|
||||
*/
|
||||
export class EmbeddingNotReadyError extends Error {
|
||||
constructor(message?: string) {
|
||||
super(message ?? "Local embedding model is not ready yet (still downloading or loading)");
|
||||
this.name = "EmbeddingNotReadyError";
|
||||
}
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Logger interface
|
||||
// ============================
|
||||
|
||||
interface Logger {
|
||||
debug?: (message: string) => void;
|
||||
info: (message: string) => void;
|
||||
warn: (message: string) => void;
|
||||
error: (message: string) => void;
|
||||
}
|
||||
|
||||
const TAG = "[memory-tdai][embedding]";
|
||||
|
||||
// ============================
|
||||
// Local (node-llama-cpp) implementation
|
||||
// ============================
|
||||
|
||||
/** Default model: Google's embeddinggemma-300m, quantized Q8_0 (~300MB) */
|
||||
const DEFAULT_LOCAL_MODEL =
|
||||
"hf:ggml-org/embeddinggemma-300m-qat-q8_0-GGUF/embeddinggemma-300m-qat-Q8_0.gguf";
|
||||
|
||||
/** embeddinggemma-300m outputs 768-dimensional vectors */
|
||||
const LOCAL_DIMENSIONS = 768;
|
||||
|
||||
/**
|
||||
* embeddinggemma-300m has a 256-token context window.
|
||||
* As a safe heuristic, we limit input to ~600 chars for CJK text
|
||||
* (CJK characters typically tokenize to 1-2 tokens each,
|
||||
* so 600 chars ≈ 200-400 tokens, keeping well within 256-token limit
|
||||
* after accounting for special tokens).
|
||||
* For Latin text, ~800 chars is a safe limit (~200 tokens).
|
||||
* We use 512 chars as a conservative universal limit.
|
||||
*/
|
||||
const LOCAL_MAX_INPUT_CHARS = 512;
|
||||
|
||||
/**
|
||||
* Sanitize NaN/Inf values and L2-normalize the vector.
|
||||
* Matches OpenClaw's own sanitizeAndNormalizeEmbedding().
|
||||
*/
|
||||
function sanitizeAndNormalize(vec: number[] | Float32Array): Float32Array {
|
||||
const arr = Array.from(vec).map((v) => (Number.isFinite(v) ? v : 0));
|
||||
const magnitude = Math.sqrt(arr.reduce((sum, v) => sum + v * v, 0));
|
||||
if (magnitude < 1e-10) {
|
||||
return new Float32Array(arr);
|
||||
}
|
||||
return new Float32Array(arr.map((v) => v / magnitude));
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialization state for LocalEmbeddingService.
|
||||
* - "idle": not started yet
|
||||
* - "initializing": model download / load is in progress (background)
|
||||
* - "ready": model is loaded and ready to serve
|
||||
* - "failed": initialization failed (will retry on next startWarmup)
|
||||
*/
|
||||
type LocalInitState = "idle" | "initializing" | "ready" | "failed";
|
||||
|
||||
/** Function that dynamically imports node-llama-cpp. Overridable for testing. */
|
||||
export type ImportLlamaFn = () => Promise<{
|
||||
getLlama: (opts: { logLevel: number }) => Promise<unknown>;
|
||||
resolveModelFile: (model: string, cacheDir?: string) => Promise<string>;
|
||||
LlamaLogLevel: { error: number };
|
||||
}>;
|
||||
|
||||
const defaultImportLlama: ImportLlamaFn = () => import("node-llama-cpp") as unknown as ReturnType<ImportLlamaFn>;
|
||||
|
||||
export class LocalEmbeddingService implements EmbeddingService {
|
||||
private readonly modelPath: string;
|
||||
private readonly modelCacheDir?: string;
|
||||
private readonly logger?: Logger;
|
||||
private readonly importLlama: ImportLlamaFn;
|
||||
|
||||
// Initialization state machine
|
||||
private initState: LocalInitState = "idle";
|
||||
private initPromise: Promise<void> | null = null;
|
||||
private initError: Error | null = null;
|
||||
private embeddingContext: {
|
||||
getEmbeddingFor: (text: string) => Promise<{ vector: Float32Array | number[] }>;
|
||||
} | null = null;
|
||||
|
||||
constructor(config?: LocalEmbeddingConfig, logger?: Logger, importLlama?: ImportLlamaFn) {
|
||||
this.modelPath = config?.modelPath?.trim() || DEFAULT_LOCAL_MODEL;
|
||||
this.modelCacheDir = config?.modelCacheDir?.trim();
|
||||
this.logger = logger;
|
||||
this.importLlama = importLlama ?? defaultImportLlama;
|
||||
}
|
||||
|
||||
getDimensions(): number {
|
||||
return LOCAL_DIMENSIONS;
|
||||
}
|
||||
|
||||
getProviderInfo(): EmbeddingProviderInfo {
|
||||
return { provider: "local", model: this.modelPath };
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether the local model is fully loaded and ready to serve requests.
|
||||
*/
|
||||
isReady(): boolean {
|
||||
return this.initState === "ready" && this.embeddingContext !== null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Start background warmup: download model (if needed) and load into memory.
|
||||
* Does NOT block the caller — returns immediately.
|
||||
* Safe to call multiple times (idempotent); re-triggers on "failed" state.
|
||||
*/
|
||||
startWarmup(): void {
|
||||
if (this.initState === "initializing" || this.initState === "ready") {
|
||||
return; // already in progress or done
|
||||
}
|
||||
this.logger?.info(`${TAG} Starting background warmup for local embedding model...`);
|
||||
this.initState = "initializing";
|
||||
this.initError = null;
|
||||
|
||||
this.initPromise = this._doInitialize()
|
||||
.then(() => {
|
||||
this.initState = "ready";
|
||||
this.logger?.info(`${TAG} Background warmup complete — local embedding ready`);
|
||||
})
|
||||
.catch((err) => {
|
||||
this.initState = "failed";
|
||||
this.initError = err instanceof Error ? err : new Error(String(err));
|
||||
this.logger?.error(
|
||||
`${TAG} Background warmup failed: ${this.initError.message}. ` +
|
||||
`embed() calls will throw EmbeddingNotReadyError until retried.`,
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Get embedding for a single text.
|
||||
* @throws {EmbeddingNotReadyError} if model is not yet ready.
|
||||
*/
|
||||
async embed(text: string, _options?: EmbeddingCallOptions): Promise<Float32Array> {
|
||||
this.assertReady();
|
||||
const truncated = this.truncateInput(text);
|
||||
const embedding = await this.embeddingContext!.getEmbeddingFor(truncated);
|
||||
return sanitizeAndNormalize(embedding.vector);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get embeddings for multiple texts.
|
||||
* @throws {EmbeddingNotReadyError} if model is not yet ready.
|
||||
*/
|
||||
async embedBatch(texts: string[], _options?: EmbeddingCallOptions): Promise<Float32Array[]> {
|
||||
if (texts.length === 0) return [];
|
||||
this.assertReady();
|
||||
|
||||
const results: Float32Array[] = [];
|
||||
for (const text of texts) {
|
||||
const truncated = this.truncateInput(text);
|
||||
const embedding = await this.embeddingContext!.getEmbeddingFor(truncated);
|
||||
results.push(sanitizeAndNormalize(embedding.vector));
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Release the node-llama-cpp embedding context and model resources.
|
||||
* Safe to call multiple times (idempotent).
|
||||
*/
|
||||
close(): void {
|
||||
if (this.embeddingContext) {
|
||||
try {
|
||||
const ctx = this.embeddingContext as unknown as { dispose?: () => void };
|
||||
ctx.dispose?.();
|
||||
} catch {
|
||||
// best-effort cleanup
|
||||
}
|
||||
this.embeddingContext = null;
|
||||
this.initPromise = null;
|
||||
this.initState = "idle";
|
||||
this.initError = null;
|
||||
this.logger?.info(`${TAG} Local embedding resources released`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Assert the model is ready. Throws EmbeddingNotReadyError if not.
|
||||
*/
|
||||
private assertReady(): void {
|
||||
if (this.initState === "ready" && this.embeddingContext) {
|
||||
return;
|
||||
}
|
||||
if (this.initState === "failed") {
|
||||
throw new EmbeddingNotReadyError(
|
||||
`Local embedding model initialization failed: ${this.initError?.message ?? "unknown error"}. ` +
|
||||
`Call startWarmup() to retry.`,
|
||||
);
|
||||
}
|
||||
if (this.initState === "initializing") {
|
||||
throw new EmbeddingNotReadyError(
|
||||
"Local embedding model is still loading (download/initialization in progress). Please try again later.",
|
||||
);
|
||||
}
|
||||
// "idle" — startWarmup() was never called
|
||||
throw new EmbeddingNotReadyError(
|
||||
"Local embedding model warmup has not been started. Call startWarmup() first.",
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Truncate input text to stay within the model's context window.
|
||||
* embeddinggemma-300m has a 256-token limit; we use a character-based
|
||||
* heuristic (LOCAL_MAX_INPUT_CHARS) as a safe proxy.
|
||||
*/
|
||||
private truncateInput(text: string): string {
|
||||
if (text.length <= LOCAL_MAX_INPUT_CHARS) return text;
|
||||
this.logger?.debug?.(
|
||||
`${TAG} Input truncated from ${text.length} to ${LOCAL_MAX_INPUT_CHARS} chars (model context limit)`,
|
||||
);
|
||||
return text.slice(0, LOCAL_MAX_INPUT_CHARS);
|
||||
}
|
||||
|
||||
/**
|
||||
* Internal: perform the actual model download + load.
|
||||
* Called by startWarmup(), runs in background.
|
||||
*/
|
||||
private async _doInitialize(): Promise<void> {
|
||||
// Track partially-initialized resources for cleanup on failure
|
||||
let model: { createEmbeddingContext: () => Promise<unknown>; dispose?: () => void } | undefined;
|
||||
try {
|
||||
this.logger?.debug?.(`${TAG} Loading node-llama-cpp for local embedding...`);
|
||||
|
||||
// Dynamic import — node-llama-cpp is a peer dependency of OpenClaw
|
||||
const { getLlama, resolveModelFile, LlamaLogLevel } = await this.importLlama();
|
||||
|
||||
const llama = await getLlama({ logLevel: LlamaLogLevel.error });
|
||||
this.logger?.debug?.(`${TAG} Llama instance created`);
|
||||
|
||||
const resolvedPath = await resolveModelFile(
|
||||
this.modelPath,
|
||||
this.modelCacheDir || undefined,
|
||||
);
|
||||
this.logger?.debug?.(`${TAG} Model resolved: ${resolvedPath}`);
|
||||
|
||||
model = await (llama as unknown as { loadModel: (opts: { modelPath: string }) => Promise<typeof model> }).loadModel({ modelPath: resolvedPath });
|
||||
this.logger?.debug?.(`${TAG} Model loaded, creating embedding context...`);
|
||||
|
||||
this.embeddingContext = await model!.createEmbeddingContext() as typeof this.embeddingContext;
|
||||
this.logger?.info(`${TAG} Local embedding ready (model=${this.modelPath}, dims=${LOCAL_DIMENSIONS})`);
|
||||
} catch (err) {
|
||||
// Clean up partially-initialized resources to prevent leaks
|
||||
if (model?.dispose) {
|
||||
try { model.dispose(); } catch { /* best-effort */ }
|
||||
}
|
||||
this.embeddingContext = null;
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait for ongoing warmup to complete (used internally by tests).
|
||||
* Returns immediately if already ready or idle.
|
||||
*/
|
||||
async waitForReady(): Promise<void> {
|
||||
if (this.initPromise) {
|
||||
await this.initPromise;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ============================
|
||||
// OpenAI-compatible implementation
|
||||
// ============================
|
||||
|
||||
/** Max texts per batch (OpenAI limit is 2048, we use a safe value) */
|
||||
const MAX_BATCH_SIZE = 256;
|
||||
|
||||
/** Max retries for API calls */
|
||||
const MAX_RETRIES = 0;
|
||||
/** Default timeout per API call in milliseconds */
|
||||
const DEFAULT_API_TIMEOUT_MS = 10_000;
|
||||
|
||||
/**
|
||||
* Custom error class for embedding API errors that carries HTTP status code.
|
||||
* Used to distinguish non-retryable client errors (4xx except 429) from
|
||||
* retryable server errors (5xx) and rate limits (429).
|
||||
*/
|
||||
class EmbeddingApiError extends Error {
|
||||
readonly httpStatus: number;
|
||||
constructor(message: string, httpStatus: number) {
|
||||
super(message);
|
||||
this.name = "EmbeddingApiError";
|
||||
this.httpStatus = httpStatus;
|
||||
}
|
||||
/** Returns true for 4xx errors that should NOT be retried (excluding 429). */
|
||||
isClientError(): boolean {
|
||||
return this.httpStatus >= 400 && this.httpStatus < 500 && this.httpStatus !== 429;
|
||||
}
|
||||
}
|
||||
|
||||
interface OpenAIEmbeddingResponse {
|
||||
data: Array<{
|
||||
index: number;
|
||||
embedding: number[];
|
||||
}>;
|
||||
usage?: {
|
||||
prompt_tokens: number;
|
||||
total_tokens: number;
|
||||
};
|
||||
}
|
||||
|
||||
export class OpenAIEmbeddingService implements EmbeddingService {
|
||||
private readonly baseUrl: string;
|
||||
private readonly apiKey: string;
|
||||
private readonly model: string;
|
||||
private readonly dims: number;
|
||||
private readonly providerName: string;
|
||||
private readonly proxyUrl?: string;
|
||||
private readonly maxInputChars?: number;
|
||||
private readonly timeoutMs: number;
|
||||
private readonly logger?: Logger;
|
||||
|
||||
constructor(config: OpenAIEmbeddingConfig, logger?: Logger) {
|
||||
if (!config.apiKey) {
|
||||
throw new Error("EmbeddingService: apiKey is required for remote provider");
|
||||
}
|
||||
if (!config.baseUrl) {
|
||||
throw new Error("EmbeddingService: baseUrl is required for remote provider");
|
||||
}
|
||||
if (!config.model) {
|
||||
throw new Error("EmbeddingService: model is required for remote provider");
|
||||
}
|
||||
if (!config.dimensions || config.dimensions <= 0) {
|
||||
throw new Error("EmbeddingService: dimensions is required for remote provider (must be a positive integer)");
|
||||
}
|
||||
this.baseUrl = config.baseUrl.replace(/\/+$/, "");
|
||||
this.apiKey = config.apiKey;
|
||||
this.model = config.model;
|
||||
this.dims = config.dimensions;
|
||||
this.providerName = config.provider || "openai";
|
||||
this.proxyUrl = config.proxyUrl?.trim() || undefined;
|
||||
this.maxInputChars = config.maxInputChars && config.maxInputChars > 0 ? config.maxInputChars : undefined;
|
||||
this.timeoutMs = config.timeoutMs && config.timeoutMs > 0 ? config.timeoutMs : DEFAULT_API_TIMEOUT_MS;
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
getDimensions(): number {
|
||||
return this.dims;
|
||||
}
|
||||
|
||||
getProviderInfo(): EmbeddingProviderInfo {
|
||||
return { provider: this.providerName, model: this.model };
|
||||
}
|
||||
|
||||
/** Remote embedding is always ready (stateless HTTP). */
|
||||
isReady(): boolean {
|
||||
return true;
|
||||
}
|
||||
|
||||
/** No-op for remote embedding (no local model to warm up). */
|
||||
startWarmup(): void {
|
||||
// nothing to do — remote API is stateless
|
||||
}
|
||||
|
||||
async embed(text: string, options?: EmbeddingCallOptions): Promise<Float32Array> {
|
||||
const [result] = await this.embedBatch([text], options);
|
||||
return result;
|
||||
}
|
||||
|
||||
async embedBatch(texts: string[], options?: EmbeddingCallOptions): Promise<Float32Array[]> {
|
||||
if (texts.length === 0) return [];
|
||||
|
||||
// Truncate texts exceeding maxInputChars limit
|
||||
const processedTexts = this.maxInputChars
|
||||
? texts.map((t) => this.truncateInput(t))
|
||||
: texts;
|
||||
|
||||
// Split into sub-batches if needed
|
||||
if (processedTexts.length > MAX_BATCH_SIZE) {
|
||||
const results: Float32Array[] = [];
|
||||
for (let i = 0; i < processedTexts.length; i += MAX_BATCH_SIZE) {
|
||||
const chunk = processedTexts.slice(i, i + MAX_BATCH_SIZE);
|
||||
const chunkResults = await this._callApi(chunk, options?.timeoutMs);
|
||||
results.push(...chunkResults);
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
return this._callApi(processedTexts, options?.timeoutMs);
|
||||
}
|
||||
|
||||
/**
|
||||
* Truncate input text to stay within the configured maxInputChars limit.
|
||||
* Logs a warning when truncation occurs.
|
||||
*/
|
||||
private truncateInput(text: string): string {
|
||||
if (!this.maxInputChars || text.length <= this.maxInputChars) return text;
|
||||
this.logger?.warn?.(
|
||||
`${TAG} Input truncated from ${text.length} to ${this.maxInputChars} chars (maxInputChars limit)`,
|
||||
);
|
||||
return text.slice(0, this.maxInputChars);
|
||||
}
|
||||
|
||||
private async _callApi(texts: string[], timeoutOverride?: number): Promise<Float32Array[]> {
|
||||
const body: Record<string, unknown> = {
|
||||
input: texts,
|
||||
model: this.model,
|
||||
dimensions: this.dims,
|
||||
};
|
||||
|
||||
// Determine fetch URL and headers based on proxy mode
|
||||
const useProxy = this.providerName === "qclaw" && !!this.proxyUrl;
|
||||
const fetchUrl = useProxy ? this.proxyUrl! : `${this.baseUrl}/embeddings`;
|
||||
const headers: Record<string, string> = {
|
||||
"Content-Type": "application/json",
|
||||
Authorization: `Bearer ${this.apiKey}`,
|
||||
};
|
||||
if (useProxy) {
|
||||
headers["Remote-URL"] = `${this.baseUrl}/embeddings`;
|
||||
this.logger?.debug?.(
|
||||
`${TAG} [qclaw-proxy] Forwarding embedding request via proxy: ${fetchUrl}, Remote-URL: ${headers["Remote-URL"]}`,
|
||||
);
|
||||
}
|
||||
|
||||
// Retry loop with timeout
|
||||
let lastError: Error | undefined;
|
||||
for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
|
||||
try {
|
||||
const controller = new AbortController();
|
||||
const timeoutId = setTimeout(() => controller.abort(), timeoutOverride ?? this.timeoutMs);
|
||||
|
||||
try {
|
||||
const resp = await fetch(fetchUrl, {
|
||||
method: "POST",
|
||||
headers,
|
||||
body: JSON.stringify(body),
|
||||
signal: controller.signal,
|
||||
});
|
||||
|
||||
if (!resp.ok) {
|
||||
const errBody = await resp.text().catch(() => "(unable to read body)");
|
||||
const err = new EmbeddingApiError(
|
||||
`Embedding API error: HTTP ${resp.status} ${resp.statusText} — ${errBody.slice(0, 500)}`,
|
||||
resp.status,
|
||||
);
|
||||
// Don't retry on 4xx client errors (except 429 rate limit)
|
||||
if (resp.status >= 400 && resp.status < 500 && resp.status !== 429) {
|
||||
throw err;
|
||||
}
|
||||
lastError = err;
|
||||
continue;
|
||||
}
|
||||
|
||||
const json = (await resp.json()) as OpenAIEmbeddingResponse;
|
||||
|
||||
if (!json.data || !Array.isArray(json.data)) {
|
||||
throw new Error("Embedding API returned unexpected format: missing 'data' array");
|
||||
}
|
||||
|
||||
// Sort by index to ensure correct order, then sanitize+normalize for consistency with local provider
|
||||
const sorted = [...json.data].sort((a, b) => a.index - b.index);
|
||||
return sorted.map((d) => sanitizeAndNormalize(d.embedding));
|
||||
} finally {
|
||||
clearTimeout(timeoutId);
|
||||
}
|
||||
} catch (err) {
|
||||
// Non-retryable errors (4xx client errors) — rethrow immediately
|
||||
if (err instanceof EmbeddingApiError && err.isClientError()) {
|
||||
throw err;
|
||||
}
|
||||
lastError = err instanceof Error ? err : new Error(String(err));
|
||||
// AbortError = timeout, retry
|
||||
if (attempt < MAX_RETRIES) {
|
||||
// Exponential backoff: 500ms, 1000ms
|
||||
const delay = 500 * (attempt + 1);
|
||||
await new Promise((r) => setTimeout(r, delay));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
throw lastError ?? new Error("Embedding API call failed after retries");
|
||||
}
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Factory
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Create an EmbeddingService from config.
|
||||
*
|
||||
* Strategy:
|
||||
* - If config has provider != "local" with valid apiKey, model, and dimensions → use remote OpenAI-compatible embedding
|
||||
* - If config has provider="local" → use node-llama-cpp local embedding
|
||||
* - If config is undefined or missing required fields → fall back to local embedding
|
||||
*
|
||||
* NOTE: For local providers, `startWarmup()` is NOT called here.
|
||||
* The caller is responsible for calling `startWarmup()` at the right time
|
||||
* (e.g. on first conversation) to avoid triggering model download during
|
||||
* short-lived CLI commands like `gateway stop` or `agents list`.
|
||||
*/
|
||||
export function createEmbeddingService(
|
||||
config: EmbeddingConfig | undefined,
|
||||
logger?: Logger,
|
||||
): EmbeddingService {
|
||||
// Remote OpenAI-compatible provider: any provider value other than "local"
|
||||
if (config && config.provider !== "local" && "apiKey" in config && config.apiKey) {
|
||||
logger?.debug?.(`${TAG} Using remote embedding (provider=${config.provider}, model=${config.model})`);
|
||||
return new OpenAIEmbeddingService(config as OpenAIEmbeddingConfig, logger);
|
||||
}
|
||||
|
||||
// Explicit local config
|
||||
if (config && config.provider === "local") {
|
||||
const localConfig = config as LocalEmbeddingConfig;
|
||||
logger?.debug?.(`${TAG} Using local embedding (node-llama-cpp, model=${localConfig.modelPath ?? DEFAULT_LOCAL_MODEL})`);
|
||||
return new LocalEmbeddingService(localConfig, logger);
|
||||
}
|
||||
|
||||
// Fallback: no config or empty apiKey → use local
|
||||
logger?.debug?.(`${TAG} No remote embedding configured, falling back to local embedding (node-llama-cpp)`);
|
||||
return new LocalEmbeddingService(undefined, logger);
|
||||
}
|
||||
|
||||
// ============================
|
||||
// NoopEmbeddingService (for server-side embedding backends)
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* No-op embedding service for backends with built-in server-side embedding
|
||||
* (e.g., TCVDB with Collection-level embedding config).
|
||||
*
|
||||
* All embed() calls return an empty Float32Array because the server generates
|
||||
* vectors automatically from the text field during upsert/search.
|
||||
*/
|
||||
export class NoopEmbeddingService implements EmbeddingService {
|
||||
embed(_text: string): Promise<Float32Array> {
|
||||
return Promise.resolve(new Float32Array(0));
|
||||
}
|
||||
|
||||
embedBatch(texts: string[]): Promise<Float32Array[]> {
|
||||
return Promise.resolve(texts.map(() => new Float32Array(0)));
|
||||
}
|
||||
|
||||
getDimensions(): number {
|
||||
return 0;
|
||||
}
|
||||
|
||||
getProviderInfo(): EmbeddingProviderInfo {
|
||||
return { provider: "noop", model: "server-side" };
|
||||
}
|
||||
|
||||
isReady(): boolean {
|
||||
return true;
|
||||
}
|
||||
|
||||
startWarmup(): void {
|
||||
// no-op
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
/**
|
||||
* Store Factory — creates the appropriate storage backend and embedding service
|
||||
* based on plugin configuration.
|
||||
*
|
||||
* Supports:
|
||||
* - "sqlite" (default): local SQLite + sqlite-vec + FTS5
|
||||
* - "tcvdb": Tencent Cloud VectorDB (server-side embedding + hybridSearch)
|
||||
*/
|
||||
|
||||
import path from "node:path";
|
||||
import type { MemoryTdaiConfig } from "../../config.js";
|
||||
import type { IMemoryStore, IEmbeddingService, StoreLogger } from "./types.js";
|
||||
import { VectorStore } from "./sqlite.js";
|
||||
import { TcvdbMemoryStore } from "./tcvdb.js";
|
||||
import { createEmbeddingService, NoopEmbeddingService } from "./embedding.js";
|
||||
import type { EmbeddingService } from "./embedding.js";
|
||||
import { createBM25Encoder } from "./bm25-local.js";
|
||||
import type { BM25LocalEncoder } from "./bm25-local.js";
|
||||
|
||||
// Re-export for convenience
|
||||
export type { IMemoryStore, IEmbeddingService, StoreLogger, BM25LocalEncoder };
|
||||
|
||||
const TAG = "[memory-tdai][factory]";
|
||||
|
||||
export interface StoreBundle {
|
||||
store: IMemoryStore;
|
||||
embedding: IEmbeddingService;
|
||||
bm25Encoder?: BM25LocalEncoder;
|
||||
/** Snapshot of current store config for manifest writing. */
|
||||
storeSnapshot: import("../../utils/manifest.js").StoreConfigSnapshot;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create the storage backend, embedding service, and optional BM25 encoder
|
||||
* based on plugin configuration.
|
||||
*
|
||||
* @param config Fully resolved plugin config.
|
||||
* @param options.dataDir Plugin data directory.
|
||||
* @param options.logger Logger instance.
|
||||
*/
|
||||
export function createStoreBundle(
|
||||
config: MemoryTdaiConfig,
|
||||
options: { dataDir: string; logger?: StoreLogger },
|
||||
): StoreBundle {
|
||||
const { logger } = options;
|
||||
|
||||
// ── BM25 local encoder ──
|
||||
const bm25Encoder = createBM25Encoder(config.bm25, logger);
|
||||
|
||||
switch (config.storeBackend) {
|
||||
case "tcvdb": {
|
||||
const tcvdbCfg = config.tcvdb;
|
||||
if (!tcvdbCfg.url || !tcvdbCfg.apiKey) {
|
||||
throw new Error(`${TAG} TCVDB backend requires tcvdb.url and tcvdb.apiKey`);
|
||||
}
|
||||
if (!tcvdbCfg.database) {
|
||||
throw new Error(`${TAG} TCVDB backend requires tcvdb.database — please set a unique database name in your openclaw.json plugin config`);
|
||||
}
|
||||
const database = tcvdbCfg.database;
|
||||
const store = new TcvdbMemoryStore({
|
||||
url: tcvdbCfg.url,
|
||||
username: tcvdbCfg.username,
|
||||
apiKey: tcvdbCfg.apiKey,
|
||||
database,
|
||||
embeddingModel: tcvdbCfg.embeddingModel,
|
||||
timeout: tcvdbCfg.timeout,
|
||||
caPemPath: tcvdbCfg.caPemPath,
|
||||
logger,
|
||||
bm25Encoder: bm25Encoder ?? undefined,
|
||||
});
|
||||
|
||||
logger?.debug?.(
|
||||
`${TAG} Store created: backend=tcvdb, database=${database}, model=${tcvdbCfg.embeddingModel}, ` +
|
||||
`bm25=${bm25Encoder ? "enabled" : "disabled"}`,
|
||||
);
|
||||
|
||||
return {
|
||||
store,
|
||||
embedding: new NoopEmbeddingService(),
|
||||
bm25Encoder,
|
||||
storeSnapshot: {
|
||||
type: "tcvdb",
|
||||
tcvdbUrl: tcvdbCfg.url,
|
||||
tcvdbDatabase: database,
|
||||
tcvdbAlias: tcvdbCfg.alias || undefined,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
case "sqlite":
|
||||
default: {
|
||||
// ── Embedding service (only when enabled) ──
|
||||
let embeddingService: EmbeddingService | undefined;
|
||||
if (config.embedding.enabled && config.embedding.provider !== "local" && config.embedding.apiKey) {
|
||||
embeddingService = createEmbeddingService({
|
||||
provider: config.embedding.provider,
|
||||
baseUrl: config.embedding.baseUrl,
|
||||
apiKey: config.embedding.apiKey,
|
||||
model: config.embedding.model,
|
||||
dimensions: config.embedding.dimensions,
|
||||
maxInputChars: config.embedding.maxInputChars,
|
||||
}, logger);
|
||||
}
|
||||
|
||||
// dimensions from config (0 when provider="none" → vec0 deferred)
|
||||
const dims = config.embedding.dimensions;
|
||||
const dbPath = path.join(options.dataDir, "vectors.db");
|
||||
const store = new VectorStore(dbPath, dims, logger);
|
||||
|
||||
logger?.debug?.(
|
||||
`${TAG} Store created: backend=sqlite, dbPath=${dbPath}, dimensions=${dims}, ` +
|
||||
`embedding=${embeddingService ? "enabled" : "disabled"}, ` +
|
||||
`bm25=${bm25Encoder ? "enabled" : "disabled"}`,
|
||||
);
|
||||
|
||||
return {
|
||||
store,
|
||||
embedding: embeddingService as unknown as IEmbeddingService,
|
||||
bm25Encoder,
|
||||
storeSnapshot: {
|
||||
type: "sqlite",
|
||||
sqlitePath: path.relative(options.dataDir, dbPath),
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
/**
|
||||
* Search utilities — shared helpers for memory search across backends.
|
||||
*
|
||||
* Contains:
|
||||
* - RRF (Reciprocal Rank Fusion) merge — used by SQLite hybrid search
|
||||
* (eliminates the 3x duplication in auto-recall, memory-search, conversation-search)
|
||||
* - FTS query building — re-exported from sqlite for convenience
|
||||
*/
|
||||
|
||||
// ============================
|
||||
// RRF (Reciprocal Rank Fusion)
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Standard RRF constant from the original RRF paper.
|
||||
* Higher k → more weight on lower-ranked items (smoother distribution).
|
||||
*/
|
||||
export const RRF_K = 60;
|
||||
|
||||
/**
|
||||
* Merge multiple ranked lists via Reciprocal Rank Fusion.
|
||||
*
|
||||
* Each item's RRF score = sum over all lists of 1/(k + rank + 1).
|
||||
* Items appearing in multiple lists get their scores summed.
|
||||
*
|
||||
* @param lists Array of ranked lists. Each list must have items with an `id` field.
|
||||
* @param k RRF constant (default: 60).
|
||||
* @returns Merged list sorted by descending RRF score, with `rrfScore` attached.
|
||||
*
|
||||
* @example
|
||||
* ```ts
|
||||
* const merged = rrfMerge(
|
||||
* [ftsResults, vecResults],
|
||||
* (item) => item.record_id,
|
||||
* );
|
||||
* ```
|
||||
*/
|
||||
export function rrfMerge<T>(
|
||||
lists: T[][],
|
||||
getId: (item: T) => string,
|
||||
k: number = RRF_K,
|
||||
): Array<T & { rrfScore: number }> {
|
||||
const map = new Map<string, { item: T; rrfScore: number }>();
|
||||
|
||||
for (const list of lists) {
|
||||
for (let rank = 0; rank < list.length; rank++) {
|
||||
const item = list[rank];
|
||||
const id = getId(item);
|
||||
const score = 1 / (k + rank + 1);
|
||||
const existing = map.get(id);
|
||||
if (existing) {
|
||||
existing.rrfScore += score;
|
||||
} else {
|
||||
map.set(id, { item, rrfScore: score });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return [...map.values()]
|
||||
.sort((a, b) => b.rrfScore - a.rrfScore)
|
||||
.map(({ item, rrfScore }) => ({ ...item, rrfScore }));
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,287 @@
|
||||
/**
|
||||
* Tencent Cloud VectorDB HTTP Client.
|
||||
*
|
||||
* Thin wrapper around the VectorDB HTTP API. Handles authentication, timeouts,
|
||||
* retries (5xx / timeout), and error normalization.
|
||||
*
|
||||
* API docs: https://cloud.tencent.com/document/product/1709
|
||||
*/
|
||||
|
||||
import fs from "node:fs";
|
||||
import { request as undiciRequest, Agent as UndiciAgent } from "undici";
|
||||
import type { Dispatcher } from "undici";
|
||||
import type { StoreLogger } from "./types.js";
|
||||
|
||||
// ============================
|
||||
// Types
|
||||
// ============================
|
||||
|
||||
export interface TcvdbClientConfig {
|
||||
/** Instance URL (e.g. "http://10.0.1.1:80") */
|
||||
url: string;
|
||||
/** Account name (default: "root") */
|
||||
username: string;
|
||||
/** API Key */
|
||||
apiKey: string;
|
||||
/** Database name */
|
||||
database: string;
|
||||
/** Request timeout in ms (default: 10000) */
|
||||
timeout: number;
|
||||
/** Path to CA certificate PEM file (for HTTPS connections) */
|
||||
caPemPath?: string;
|
||||
}
|
||||
|
||||
/** Standard VectorDB API response envelope. */
|
||||
interface ApiResponse {
|
||||
code: number;
|
||||
msg: string;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
/** Search/hybridSearch response shape. */
|
||||
export interface SearchResponse {
|
||||
documents: Array<Array<Record<string, unknown>>>;
|
||||
}
|
||||
|
||||
/** Query response shape. */
|
||||
export interface QueryResponse {
|
||||
documents: Array<Record<string, unknown>>;
|
||||
count?: number;
|
||||
}
|
||||
|
||||
/** Collection info from describeCollection. */
|
||||
export interface CollectionInfo {
|
||||
collection: string;
|
||||
database: string;
|
||||
documentCount?: number;
|
||||
embedding?: {
|
||||
field: string;
|
||||
vectorField: string;
|
||||
model: string;
|
||||
};
|
||||
indexes?: Array<Record<string, unknown>>;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
export class TcvdbApiError extends Error {
|
||||
readonly apiCode: number;
|
||||
constructor(path: string, code: number, msg: string) {
|
||||
super(`VectorDB ${path}: code=${code}, msg=${msg}`);
|
||||
this.name = "TcvdbApiError";
|
||||
this.apiCode = code;
|
||||
}
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Client
|
||||
// ============================
|
||||
|
||||
const TAG = "[memory-tdai][tcvdb-client]";
|
||||
const MAX_RETRIES = 2;
|
||||
|
||||
export class TcvdbClient {
|
||||
private readonly baseUrl: string;
|
||||
private readonly authHeader: string;
|
||||
private readonly database: string;
|
||||
private readonly timeout: number;
|
||||
private readonly logger?: StoreLogger;
|
||||
/** undici dispatcher for HTTPS + custom CA. */
|
||||
private readonly dispatcher?: Dispatcher;
|
||||
|
||||
constructor(config: TcvdbClientConfig, logger?: StoreLogger) {
|
||||
this.baseUrl = config.url.replace(/\/+$/, "");
|
||||
this.authHeader = `Bearer account=${config.username}&api_key=${config.apiKey}`;
|
||||
this.database = config.database;
|
||||
this.timeout = config.timeout;
|
||||
this.logger = logger;
|
||||
|
||||
// Log connection info at construction time.
|
||||
this.logger?.debug?.(`${TAG} url=${this.baseUrl} db=${this.database} timeout=${this.timeout}${this.baseUrl.startsWith("https://") ? ` https=true caPemPath=${config.caPemPath ?? "(none)"}` : ""}`);
|
||||
|
||||
// For HTTPS with a custom CA certificate, create a dedicated undici Agent.
|
||||
// We use undici.request() instead of global fetch because fetch's
|
||||
// `dispatcher` option is unreliable across Node versions.
|
||||
if (this.baseUrl.startsWith("https://") && config.caPemPath) {
|
||||
try {
|
||||
const ca = fs.readFileSync(config.caPemPath, "utf-8");
|
||||
this.dispatcher = new UndiciAgent({ connect: { ca } });
|
||||
this.logger?.debug?.(`${TAG} HTTPS enabled with CA from ${config.caPemPath}`);
|
||||
} catch (err) {
|
||||
this.logger?.error(`${TAG} Failed to load CA PEM from ${config.caPemPath}: ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── Generic request ─────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Send a POST request to VectorDB API.
|
||||
* Handles auth, timeout, retries (5xx/timeout), and error unwrapping.
|
||||
*/
|
||||
async request<T = ApiResponse>(path: string, body: Record<string, unknown>): Promise<T> {
|
||||
let lastError: Error | undefined;
|
||||
|
||||
for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
|
||||
try {
|
||||
this.logger?.debug?.(`${TAG} → ${path} body=${JSON.stringify(body).slice(0, 500)}`);
|
||||
const { statusCode, body: respBody } = await undiciRequest(`${this.baseUrl}${path}`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": this.authHeader,
|
||||
},
|
||||
body: JSON.stringify(body),
|
||||
signal: AbortSignal.timeout(this.timeout),
|
||||
...(this.dispatcher ? { dispatcher: this.dispatcher } : {}),
|
||||
});
|
||||
|
||||
const text = await respBody.text();
|
||||
const json = JSON.parse(text) as ApiResponse;
|
||||
this.logger?.debug?.(`${TAG} ← ${path} status=${statusCode} code=${json.code} msg=${json.msg} keys=[${Object.keys(json).join(",")}]`);
|
||||
|
||||
if (json.code !== 0) {
|
||||
const err = new TcvdbApiError(path, json.code, json.msg);
|
||||
if (statusCode !== undefined && statusCode >= 400 && statusCode < 500) throw err;
|
||||
lastError = err;
|
||||
continue;
|
||||
}
|
||||
|
||||
return json as unknown as T;
|
||||
} catch (err) {
|
||||
if (err instanceof TcvdbApiError && err.apiCode !== 0) throw err;
|
||||
lastError = err instanceof Error ? err : new Error(String(err));
|
||||
if (attempt < MAX_RETRIES) {
|
||||
const delay = 500 * (attempt + 1);
|
||||
this.logger?.debug?.(`${TAG} ${path} retry ${attempt + 1}/${MAX_RETRIES} in ${delay}ms`);
|
||||
await new Promise((r) => setTimeout(r, delay));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
throw lastError ?? new Error(`${TAG} ${path} failed after retries`);
|
||||
}
|
||||
|
||||
// ── Database operations ─────────────────────────────────
|
||||
|
||||
async createDatabase(dbName?: string): Promise<boolean> {
|
||||
const name = dbName ?? this.database;
|
||||
// SDK pattern: list first, create only if not found
|
||||
const listResp = await this.request<{ databases: string[] }>("/database/list", {});
|
||||
const exists = (listResp.databases ?? []).includes(name);
|
||||
if (exists) {
|
||||
this.logger?.debug?.(`${TAG} Database already exists: ${name}`);
|
||||
return false;
|
||||
}
|
||||
await this.request("/database/create", { database: name });
|
||||
this.logger?.info(`${TAG} Database created: ${name}`);
|
||||
return true;
|
||||
}
|
||||
|
||||
// ── Collection operations ───────────────────────────────
|
||||
|
||||
async createCollection(params: Record<string, unknown>): Promise<void> {
|
||||
const name = String(params.collection ?? "");
|
||||
// SDK pattern: try describe first, create only if not found (code 15302)
|
||||
try {
|
||||
await this.describeCollection(name);
|
||||
this.logger?.debug?.(`${TAG} Collection already exists: ${name}`);
|
||||
return;
|
||||
} catch (err) {
|
||||
if (!(err instanceof TcvdbApiError && err.apiCode === 15302)) {
|
||||
throw err; // unexpected error
|
||||
}
|
||||
// 15302 = collection not found → proceed to create
|
||||
}
|
||||
try {
|
||||
await this.request("/collection/create", {
|
||||
database: this.database,
|
||||
...params,
|
||||
});
|
||||
this.logger?.info(`${TAG} Collection created: ${name}`);
|
||||
} catch (err) {
|
||||
// 15202 = collection already exists — race between describe and create.
|
||||
// Semantically identical to "describe found it", so treat as success.
|
||||
if (err instanceof TcvdbApiError && err.apiCode === 15202) {
|
||||
this.logger?.debug?.(`${TAG} Collection already exists (race): ${name}`);
|
||||
return;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
async describeCollection(collection: string): Promise<CollectionInfo> {
|
||||
const resp = await this.request<{ collection: CollectionInfo }>("/collection/describe", {
|
||||
database: this.database,
|
||||
collection,
|
||||
});
|
||||
return resp.collection;
|
||||
}
|
||||
|
||||
// ── Document operations ─────────────────────────────────
|
||||
|
||||
async upsert(collection: string, documents: Record<string, unknown>[]): Promise<void> {
|
||||
await this.request("/document/upsert", {
|
||||
database: this.database,
|
||||
collection,
|
||||
buildIndex: true,
|
||||
documents,
|
||||
});
|
||||
}
|
||||
|
||||
async search(collection: string, searchParams: Record<string, unknown>): Promise<SearchResponse> {
|
||||
return this.request<SearchResponse>("/document/search", {
|
||||
database: this.database,
|
||||
collection,
|
||||
readConsistency: "strongConsistency",
|
||||
search: searchParams,
|
||||
});
|
||||
}
|
||||
|
||||
async hybridSearch(collection: string, searchParams: Record<string, unknown>): Promise<SearchResponse> {
|
||||
return this.request<SearchResponse>("/document/hybridSearch", {
|
||||
database: this.database,
|
||||
collection,
|
||||
readConsistency: "strongConsistency",
|
||||
search: searchParams,
|
||||
});
|
||||
}
|
||||
|
||||
async query(collection: string, queryParams: Record<string, unknown>): Promise<QueryResponse> {
|
||||
return this.request<QueryResponse>("/document/query", {
|
||||
database: this.database,
|
||||
collection,
|
||||
readConsistency: "strongConsistency",
|
||||
query: queryParams,
|
||||
});
|
||||
}
|
||||
|
||||
async deleteDoc(collection: string, params: Record<string, unknown>): Promise<void> {
|
||||
await this.request("/document/delete", {
|
||||
database: this.database,
|
||||
collection,
|
||||
...params,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Count documents matching an optional filter.
|
||||
* Uses the dedicated /document/count endpoint.
|
||||
*/
|
||||
async count(collection: string, filter?: string): Promise<number> {
|
||||
const query: Record<string, unknown> = {};
|
||||
if (filter) query.filter = filter;
|
||||
const resp = await this.request<{ count: number }>("/document/count", {
|
||||
database: this.database,
|
||||
collection,
|
||||
readConsistency: "strongConsistency",
|
||||
query,
|
||||
});
|
||||
return resp.count ?? 0;
|
||||
}
|
||||
|
||||
// ── Convenience getters ─────────────────────────────────
|
||||
|
||||
getDatabase(): string {
|
||||
return this.database;
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,328 @@
|
||||
/**
|
||||
* Memory Store Abstraction Layer — Core Types & Interfaces.
|
||||
*
|
||||
* This module defines the storage contracts that all backend implementations
|
||||
* (SQLite local, Tencent Cloud VectorDB, etc.) must satisfy.
|
||||
*
|
||||
* Design principles:
|
||||
* 1. **Backend-agnostic**: Upper-layer modules (hooks, tools, pipeline, record)
|
||||
* depend only on these interfaces — never on concrete implementations.
|
||||
* 2. **Capability-based**: Features like vector search, FTS, and hybrid search
|
||||
* are expressed as capability flags so callers can gracefully degrade.
|
||||
* 3. **Fault-tolerant**: All methods return empty results or `false` on
|
||||
* failure rather than throwing, unless explicitly documented otherwise.
|
||||
* 4. **Sync-first**: Matches current SQLite DatabaseSync usage. TCVDB backend
|
||||
* adapts internally without changing these signatures.
|
||||
*/
|
||||
|
||||
import type { MemoryRecord } from "../record/l1-writer.js";
|
||||
import type { EmbeddingProviderInfo } from "./embedding.js";
|
||||
|
||||
// Re-export so consumers can import everything from types.ts
|
||||
export type { MemoryRecord, EmbeddingProviderInfo };
|
||||
|
||||
// ============================
|
||||
// Common Types
|
||||
// ============================
|
||||
|
||||
/** Minimal logger interface accepted by store implementations. */
|
||||
export interface StoreLogger {
|
||||
debug?: (message: string) => void;
|
||||
info: (message: string) => void;
|
||||
warn: (message: string) => void;
|
||||
error: (message: string) => void;
|
||||
}
|
||||
|
||||
// ============================
|
||||
// L1 Types (Structured Memories)
|
||||
// ============================
|
||||
|
||||
/** Result from an L1 vector similarity search. */
|
||||
export interface L1SearchResult {
|
||||
record_id: string;
|
||||
content: string;
|
||||
type: string;
|
||||
priority: number;
|
||||
scene_name: string;
|
||||
/** Similarity score (0–1, higher is better). */
|
||||
score: number;
|
||||
timestamp_str: string;
|
||||
timestamp_start: string;
|
||||
timestamp_end: string;
|
||||
session_key: string;
|
||||
session_id: string;
|
||||
metadata_json: string;
|
||||
}
|
||||
|
||||
/** Result from an L1 FTS keyword search. */
|
||||
export interface L1FtsResult {
|
||||
record_id: string;
|
||||
content: string;
|
||||
type: string;
|
||||
priority: number;
|
||||
scene_name: string;
|
||||
/** BM25-derived score (0–1, higher is better). */
|
||||
score: number;
|
||||
timestamp_str: string;
|
||||
timestamp_start: string;
|
||||
timestamp_end: string;
|
||||
session_key: string;
|
||||
session_id: string;
|
||||
metadata_json: string;
|
||||
}
|
||||
|
||||
/** Filter options for querying L1 records. */
|
||||
export interface L1QueryFilter {
|
||||
sessionKey?: string;
|
||||
sessionId?: string;
|
||||
/** Only return records with updated_time strictly after this ISO 8601 UTC timestamp. */
|
||||
updatedAfter?: string;
|
||||
}
|
||||
|
||||
/** Row shape returned by L1 query methods. */
|
||||
export interface L1RecordRow {
|
||||
record_id: string;
|
||||
content: string;
|
||||
type: string;
|
||||
priority: number;
|
||||
scene_name: string;
|
||||
session_key: string;
|
||||
session_id: string;
|
||||
timestamp_str: string;
|
||||
timestamp_start: string;
|
||||
timestamp_end: string;
|
||||
created_time: string;
|
||||
updated_time: string;
|
||||
metadata_json: string;
|
||||
}
|
||||
|
||||
// ============================
|
||||
// L0 Types (Raw Conversations)
|
||||
// ============================
|
||||
|
||||
/** An L0 conversation message record for vector indexing. */
|
||||
export interface L0Record {
|
||||
id: string;
|
||||
sessionKey: string;
|
||||
sessionId: string;
|
||||
role: string;
|
||||
messageText: string;
|
||||
recordedAt: string;
|
||||
/** Original message timestamp (epoch ms). */
|
||||
timestamp: number;
|
||||
}
|
||||
|
||||
/** Result from an L0 vector similarity search. */
|
||||
export interface L0SearchResult {
|
||||
record_id: string;
|
||||
session_key: string;
|
||||
session_id: string;
|
||||
role: string;
|
||||
message_text: string;
|
||||
/** Similarity score (0–1, higher is better). */
|
||||
score: number;
|
||||
recorded_at: string;
|
||||
timestamp: number;
|
||||
}
|
||||
|
||||
/** Result from an L0 FTS keyword search. */
|
||||
export interface L0FtsResult {
|
||||
record_id: string;
|
||||
session_key: string;
|
||||
session_id: string;
|
||||
role: string;
|
||||
message_text: string;
|
||||
/** BM25-derived score (0–1, higher is better). */
|
||||
score: number;
|
||||
recorded_at: string;
|
||||
timestamp: number;
|
||||
}
|
||||
|
||||
/** Raw L0 row returned by query methods (used by L1 runner). */
|
||||
export interface L0QueryRow {
|
||||
record_id: string;
|
||||
session_key: string;
|
||||
session_id: string;
|
||||
role: string;
|
||||
message_text: string;
|
||||
recorded_at: string;
|
||||
timestamp: number;
|
||||
}
|
||||
|
||||
/** L0 messages grouped by session ID (for L1 runner). */
|
||||
export interface L0SessionGroup {
|
||||
sessionId: string;
|
||||
messages: Array<{
|
||||
id: string;
|
||||
role: string;
|
||||
content: string;
|
||||
timestamp: number;
|
||||
/** Epoch ms when this message was recorded into L0 (used by L1 cursor). */
|
||||
recordedAtMs: number;
|
||||
}>;
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Store Init Result
|
||||
// ============================
|
||||
|
||||
/** Result of store initialization. */
|
||||
export interface StoreInitResult {
|
||||
/** Whether embeddings need to be regenerated (provider/model change). */
|
||||
needsReindex: boolean;
|
||||
/** Human-readable reason (for logging). */
|
||||
reason?: string;
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Capability Flags
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Describes what search capabilities a store backend supports.
|
||||
* Callers use this to select search strategies and degrade gracefully.
|
||||
*/
|
||||
export interface StoreCapabilities {
|
||||
/** Whether vector (embedding) search is available. */
|
||||
vectorSearch: boolean;
|
||||
/** Whether FTS (full-text keyword) search is available. */
|
||||
ftsSearch: boolean;
|
||||
/** Whether native hybrid search is supported (e.g., TCVDB hybridSearch). */
|
||||
nativeHybridSearch: boolean;
|
||||
/** Whether the store supports sparse vectors (BM25 encoding). */
|
||||
sparseVectors: boolean;
|
||||
}
|
||||
|
||||
// ============================
|
||||
// L2/L3 Profile Sync Types
|
||||
// ============================
|
||||
|
||||
/** Canonical L2/L3 profile row shared between local cache and remote store. */
|
||||
export interface ProfileRecord {
|
||||
/** Stable ID: `profile:v1:${sha256(scope + "\0" + type + "\0" + filename)}`. */
|
||||
id: string;
|
||||
type: "l2" | "l3";
|
||||
filename: string;
|
||||
content: string;
|
||||
contentMd5: string;
|
||||
agentId?: string;
|
||||
version: number;
|
||||
createdAtMs: number;
|
||||
updatedAtMs: number;
|
||||
}
|
||||
|
||||
/** Profile upsert payload with optimistic-lock baseline from the last pull. */
|
||||
export interface ProfileSyncRecord extends ProfileRecord {
|
||||
baselineVersion?: number;
|
||||
}
|
||||
|
||||
// ============================
|
||||
// IMemoryStore — The Core Abstraction
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Unified memory store interface.
|
||||
*
|
||||
* Implementations:
|
||||
* - `SqliteMemoryStore` (sqlite.ts) — local SQLite + sqlite-vec + FTS5
|
||||
* - `TcvdbMemoryStore` (tcvdb.ts) — Tencent Cloud VectorDB (future)
|
||||
*
|
||||
* All methods are fault-tolerant: they return empty results or `false` on
|
||||
* failure rather than throwing, unless explicitly documented otherwise.
|
||||
*/
|
||||
/**
|
||||
* Helper type: a value that may be sync or async.
|
||||
* Callers should always `await` the result — it's safe for both sync and async values.
|
||||
*/
|
||||
export type MaybePromise<T> = T | Promise<T>;
|
||||
|
||||
export interface IMemoryStore {
|
||||
// ── Capabilities ───────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Whether this store supports deferred (background) embedding updates.
|
||||
*
|
||||
* When `true`, auto-capture writes metadata-only via `upsertL0(record, undefined)`
|
||||
* and later calls `updateL0Embedding()` in a fire-and-forget background task.
|
||||
* When `false` or absent, embedding is computed inline and passed to `upsertL0()`.
|
||||
*/
|
||||
readonly supportsDeferredEmbedding?: boolean;
|
||||
|
||||
// ── Lifecycle (always sync) ──────────────────────────────
|
||||
|
||||
init(providerInfo?: EmbeddingProviderInfo): MaybePromise<StoreInitResult>;
|
||||
isDegraded(): boolean;
|
||||
getCapabilities(): StoreCapabilities;
|
||||
close(): void;
|
||||
|
||||
// ── L1 Write ─────────────────────────────────────────────
|
||||
|
||||
upsertL1(record: MemoryRecord, embedding?: Float32Array): MaybePromise<boolean>;
|
||||
deleteL1(recordId: string): MaybePromise<boolean>;
|
||||
deleteL1Batch(recordIds: string[]): MaybePromise<boolean>;
|
||||
deleteL1Expired(cutoffIso: string): MaybePromise<number>;
|
||||
|
||||
// ── L1 Read ──────────────────────────────────────────────
|
||||
|
||||
countL1(): MaybePromise<number>;
|
||||
queryL1Records(filter?: L1QueryFilter): MaybePromise<L1RecordRow[]>;
|
||||
getAllL1Texts(): MaybePromise<Array<{ record_id: string; content: string; updated_time: string }>>;
|
||||
|
||||
// ── L1 Search ────────────────────────────────────────────
|
||||
|
||||
searchL1Vector(queryEmbedding: Float32Array, topK?: number, queryText?: string): MaybePromise<L1SearchResult[]>;
|
||||
searchL1Fts(ftsQuery: string, limit?: number): MaybePromise<L1FtsResult[]>;
|
||||
searchL1Hybrid?(params: {
|
||||
query?: string;
|
||||
queryEmbedding?: Float32Array;
|
||||
sparseVector?: Array<[number, number]>;
|
||||
topK?: number;
|
||||
}): MaybePromise<L1SearchResult[]>;
|
||||
|
||||
// ── L0 Write ─────────────────────────────────────────────
|
||||
|
||||
upsertL0(record: L0Record, embedding?: Float32Array): MaybePromise<boolean>;
|
||||
/** Update only the vector embedding for an existing L0 record (sqlite background path). */
|
||||
updateL0Embedding?(recordId: string, embedding: Float32Array): MaybePromise<boolean>;
|
||||
deleteL0(recordId: string): MaybePromise<boolean>;
|
||||
deleteL0Expired(cutoffIso: string): MaybePromise<number>;
|
||||
|
||||
// ── L0 Read ──────────────────────────────────────────────
|
||||
|
||||
countL0(): MaybePromise<number>;
|
||||
queryL0ForL1(sessionKey: string, afterRecordedAtMs?: number, limit?: number): MaybePromise<L0QueryRow[]>;
|
||||
queryL0GroupedBySessionId(sessionKey: string, afterRecordedAtMs?: number, limit?: number): MaybePromise<L0SessionGroup[]>;
|
||||
getAllL0Texts(): MaybePromise<Array<{ record_id: string; message_text: string; recorded_at: string }>>;
|
||||
|
||||
// ── L0 Search ────────────────────────────────────────────
|
||||
|
||||
searchL0Vector(queryEmbedding: Float32Array, topK?: number, queryText?: string): MaybePromise<L0SearchResult[]>;
|
||||
searchL0Fts(ftsQuery: string, limit?: number): MaybePromise<L0FtsResult[]>;
|
||||
|
||||
pullProfiles?(): Promise<ProfileRecord[]>;
|
||||
syncProfiles?(records: ProfileSyncRecord[]): Promise<void>;
|
||||
deleteProfiles?(recordIds: string[]): Promise<void>;
|
||||
|
||||
// ── Re-index ─────────────────────────────────────────────
|
||||
|
||||
reindexAll(
|
||||
embedFn: (text: string) => Promise<Float32Array>,
|
||||
onProgress?: (done: number, total: number, layer: "L1" | "L0") => void,
|
||||
): Promise<{ l1Count: number; l0Count: number }>;
|
||||
|
||||
// ── FTS (always sync — cached flag) ──────────────────────
|
||||
|
||||
isFtsAvailable(): boolean;
|
||||
}
|
||||
|
||||
// ============================
|
||||
// IEmbeddingService — re-exported from embedding.ts for convenience
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Re-export EmbeddingService as IEmbeddingService for backward compatibility.
|
||||
* The canonical definition lives in `./embedding.ts`. All concrete implementations
|
||||
* (LocalEmbeddingService, OpenAIEmbeddingService, NoopEmbeddingService) implement
|
||||
* the EmbeddingService interface from embedding.ts.
|
||||
*/
|
||||
export type { EmbeddingService as IEmbeddingService } from "./embedding.js";
|
||||
@@ -0,0 +1,534 @@
|
||||
/**
|
||||
* TdaiCore — Host-neutral facade for TDAI memory capabilities.
|
||||
*
|
||||
* This is the single entry point that both OpenClaw and Hermes/Gateway call
|
||||
* to perform recall, capture, search, and pipeline management. It depends
|
||||
* only on abstract interfaces (HostAdapter, LLMRunner), never on a specific host.
|
||||
*
|
||||
* Usage:
|
||||
* // OpenClaw path (in-process)
|
||||
* const adapter = new OpenClawHostAdapter({ api, pluginDataDir, config });
|
||||
* const core = new TdaiCore({ hostAdapter: adapter, config: parsedCfg });
|
||||
* await core.initialize();
|
||||
* const recall = await core.handleBeforeRecall("user query", "session-1");
|
||||
*
|
||||
* // Gateway path (HTTP)
|
||||
* const adapter = new StandaloneHostAdapter({ ... });
|
||||
* const core = new TdaiCore({ hostAdapter: adapter, config: parsedCfg });
|
||||
* await core.initialize();
|
||||
* // HTTP handler calls core.handleBeforeRecall / core.handleTurnCommitted / etc.
|
||||
*/
|
||||
|
||||
import type {
|
||||
HostAdapter,
|
||||
Logger,
|
||||
LLMRunnerFactory,
|
||||
RecallResult,
|
||||
CaptureResult,
|
||||
CompletedTurn,
|
||||
MemorySearchParams,
|
||||
ConversationSearchParams,
|
||||
} from "./types.js";
|
||||
import type { MemoryTdaiConfig } from "../config.js";
|
||||
import type { IMemoryStore } from "./store/types.js";
|
||||
import type { EmbeddingService } from "./store/embedding.js";
|
||||
import { performAutoRecall } from "./hooks/auto-recall.js";
|
||||
import { performAutoCapture } from "./hooks/auto-capture.js";
|
||||
import { executeMemorySearch, formatSearchResponse } from "./tools/memory-search.js";
|
||||
import { executeConversationSearch, formatConversationSearchResponse } from "./tools/conversation-search.js";
|
||||
import {
|
||||
initDataDirectories,
|
||||
initStores,
|
||||
resetStores,
|
||||
createPipelineManager,
|
||||
createL1Runner,
|
||||
createPersister,
|
||||
createL2Runner,
|
||||
createL3Runner,
|
||||
} from "../utils/pipeline-factory.js";
|
||||
import { MemoryPipelineManager } from "../utils/pipeline-manager.js";
|
||||
import { CheckpointManager } from "../utils/checkpoint.js";
|
||||
import { SessionFilter } from "../utils/session-filter.js";
|
||||
import { StandaloneLLMRunnerFactory } from "../adapters/standalone/llm-runner.js";
|
||||
|
||||
const TAG = "[memory-tdai] [core]";
|
||||
|
||||
// ============================
|
||||
// Constructor options
|
||||
// ============================
|
||||
|
||||
export interface TdaiCoreOptions {
|
||||
/** Host adapter providing runtime context, logger, and LLM runner factory. */
|
||||
hostAdapter: HostAdapter;
|
||||
/** Parsed TDAI memory configuration. */
|
||||
config: MemoryTdaiConfig;
|
||||
/** Session filter for excluding internal/benchmark sessions. */
|
||||
sessionFilter?: SessionFilter;
|
||||
/** Plugin instance ID for metric reporting. */
|
||||
instanceId?: string;
|
||||
}
|
||||
|
||||
// ============================
|
||||
// TdaiCore
|
||||
// ============================
|
||||
|
||||
export class TdaiCore {
|
||||
private hostAdapter: HostAdapter;
|
||||
private cfg: MemoryTdaiConfig;
|
||||
private logger: Logger;
|
||||
private dataDir: string;
|
||||
private runnerFactory: LLMRunnerFactory;
|
||||
private sessionFilter: SessionFilter;
|
||||
private instanceId?: string;
|
||||
|
||||
// Lazy-initialized resources
|
||||
private vectorStore?: IMemoryStore;
|
||||
private embeddingService?: EmbeddingService;
|
||||
private scheduler?: MemoryPipelineManager;
|
||||
/**
|
||||
* Promise gate for the one-shot scheduler-start sequence.
|
||||
*
|
||||
* ``ensureSchedulerStarted`` reads a checkpoint file (async) and then
|
||||
* calls ``scheduler.start(restoredStates)``. Under the Gateway, several
|
||||
* HTTP requests can reach ``handleTurnCommitted`` concurrently and all
|
||||
* race into that function. Using a plain boolean flag is unsafe: the
|
||||
* first caller flips the flag to ``true`` *before* the await completes,
|
||||
* so subsequent callers slip past the check and touch the scheduler
|
||||
* before ``start()`` has actually run — which makes ``start()``'s
|
||||
* ``sessionStates.set(key, restored)`` later clobber the state that
|
||||
* those concurrent captures already incremented.
|
||||
*
|
||||
* Storing the in-flight promise lets every concurrent caller ``await``
|
||||
* the same start sequence. Once it resolves the promise is kept as a
|
||||
* sentinel so subsequent calls are a single already-resolved await
|
||||
* (effectively a no-op).
|
||||
*/
|
||||
private schedulerStartPromise?: Promise<void>;
|
||||
private storeReady?: Promise<void>;
|
||||
|
||||
/**
|
||||
* In-flight fire-and-forget background tasks started by
|
||||
* ``handleTurnCommitted`` (currently: deferred L0 embedding for
|
||||
* SQLite-style stores — see auto-capture.ts path A).
|
||||
*
|
||||
* ``destroy()`` awaits all pending entries (with a hard timeout)
|
||||
* before closing ``vectorStore`` / ``embeddingService`` so that a
|
||||
* late ``updateL0Embedding`` cannot land on an already-closed
|
||||
* database connection.
|
||||
*
|
||||
* Each task registers itself on creation and removes itself in its
|
||||
* own ``finally`` handler, so the set stays bounded by the number
|
||||
* of currently-running background tasks.
|
||||
*/
|
||||
private readonly bgTasks = new Set<Promise<void>>();
|
||||
|
||||
constructor(opts: TdaiCoreOptions) {
|
||||
this.hostAdapter = opts.hostAdapter;
|
||||
this.cfg = opts.config;
|
||||
this.logger = opts.hostAdapter.getLogger();
|
||||
this.dataDir = opts.hostAdapter.getRuntimeContext().dataDir;
|
||||
this.runnerFactory = opts.hostAdapter.getLLMRunnerFactory();
|
||||
this.sessionFilter = opts.sessionFilter ?? new SessionFilter([]);
|
||||
this.instanceId = opts.instanceId;
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Lifecycle
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Initialize data directories, storage, and pipeline scheduler.
|
||||
* Must be called once before any other methods.
|
||||
*/
|
||||
async initialize(): Promise<void> {
|
||||
this.logger.debug?.(`${TAG} Initializing TDAI Core: dataDir=${this.dataDir}`);
|
||||
initDataDirectories(this.dataDir);
|
||||
|
||||
// Initialize stores (async)
|
||||
this.storeReady = this.initStores();
|
||||
|
||||
// Create pipeline manager (sync — does not need store)
|
||||
if (this.cfg.extraction.enabled) {
|
||||
this.scheduler = createPipelineManager(this.cfg, this.logger, this.sessionFilter);
|
||||
// Wire runners after store is ready (or after store init fails — runners
|
||||
// still work in degraded mode with JSONL fallback and no embedding)
|
||||
this.storeReady
|
||||
.then(() => this.wirePipelineRunners())
|
||||
.catch((err) => {
|
||||
this.logger.error(`${TAG} Store init failed; wiring pipeline runners in degraded mode: ${err instanceof Error ? err.message : String(err)}`);
|
||||
this.wirePipelineRunners();
|
||||
});
|
||||
}
|
||||
|
||||
this.logger.debug?.(`${TAG} TDAI Core initialized`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Destroy all resources. Call on shutdown.
|
||||
*/
|
||||
async destroy(): Promise<void> {
|
||||
this.logger.debug?.(`${TAG} Destroying TDAI Core...`);
|
||||
|
||||
// Wait for store init to complete before tearing down
|
||||
await this.storeReady?.catch(() => {});
|
||||
|
||||
if (this.scheduler && this.schedulerStartPromise) {
|
||||
await this.scheduler.destroy();
|
||||
this.schedulerStartPromise = undefined;
|
||||
this.logger.debug?.(`${TAG} Scheduler destroyed`);
|
||||
}
|
||||
|
||||
// Drain fire-and-forget background tasks started by auto-capture
|
||||
// (currently: deferred L0 embedding writes). We must wait for
|
||||
// them here — BEFORE closing vectorStore / embeddingService —
|
||||
// otherwise a late updateL0Embedding lands on an already-closed
|
||||
// DB connection and either throws "database is not open" or
|
||||
// (worse) corrupts state. A hard timeout keeps destroy bounded
|
||||
// when a background task is stuck on a hung embed HTTP call.
|
||||
if (this.bgTasks.size > 0) {
|
||||
const pending = [...this.bgTasks];
|
||||
this.logger.debug?.(
|
||||
`${TAG} Draining ${pending.length} background task(s) before closing stores...`,
|
||||
);
|
||||
const BG_DRAIN_TIMEOUT_MS = 5_000;
|
||||
let drainTimeoutId: ReturnType<typeof setTimeout> | undefined;
|
||||
try {
|
||||
await Promise.race([
|
||||
Promise.allSettled(pending).then(() => undefined),
|
||||
new Promise<never>((_, reject) => {
|
||||
drainTimeoutId = setTimeout(
|
||||
() => reject(new Error("bgTasks drain timeout")),
|
||||
BG_DRAIN_TIMEOUT_MS,
|
||||
);
|
||||
}),
|
||||
]);
|
||||
this.logger.debug?.(`${TAG} Background tasks drained`);
|
||||
} catch (err) {
|
||||
this.logger.warn(
|
||||
`${TAG} Background-task drain timed out (${BG_DRAIN_TIMEOUT_MS}ms): ` +
|
||||
`${err instanceof Error ? err.message : String(err)}. ` +
|
||||
`Closing stores anyway — residual writes may surface as warnings.`,
|
||||
);
|
||||
} finally {
|
||||
if (drainTimeoutId !== undefined) clearTimeout(drainTimeoutId);
|
||||
}
|
||||
}
|
||||
|
||||
if (this.vectorStore) {
|
||||
this.vectorStore.close();
|
||||
this.vectorStore = undefined;
|
||||
this.logger.debug?.(`${TAG} VectorStore closed`);
|
||||
}
|
||||
|
||||
if (this.embeddingService?.close) {
|
||||
try {
|
||||
await this.embeddingService.close();
|
||||
} catch (err) {
|
||||
this.logger.warn(`${TAG} EmbeddingService close error: ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
this.embeddingService = undefined;
|
||||
}
|
||||
|
||||
resetStores(this.dataDir);
|
||||
this.logger.debug?.(`${TAG} TDAI Core destroyed`);
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Core capabilities
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Handle recall (memory retrieval) before an LLM turn.
|
||||
* Maps to: OpenClaw `before_prompt_build` / Hermes `prefetch()`.
|
||||
*/
|
||||
async handleBeforeRecall(userText: string, sessionKey: string): Promise<RecallResult> {
|
||||
await this.storeReady?.catch(() => {});
|
||||
|
||||
const result = await performAutoRecall({
|
||||
userText,
|
||||
actorId: "default_user",
|
||||
sessionKey,
|
||||
cfg: this.cfg,
|
||||
pluginDataDir: this.dataDir,
|
||||
logger: this.logger,
|
||||
vectorStore: this.vectorStore,
|
||||
embeddingService: this.embeddingService,
|
||||
});
|
||||
|
||||
return result ?? {};
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle turn commitment (conversation capture + pipeline trigger).
|
||||
* Maps to: OpenClaw `agent_end` / Hermes `sync_turn()`.
|
||||
*/
|
||||
async handleTurnCommitted(turn: CompletedTurn): Promise<CaptureResult> {
|
||||
await this.storeReady?.catch(() => {});
|
||||
await this.ensureSchedulerStarted();
|
||||
|
||||
return performAutoCapture({
|
||||
messages: turn.messages,
|
||||
sessionKey: turn.sessionKey,
|
||||
sessionId: turn.sessionId,
|
||||
cfg: this.cfg,
|
||||
pluginDataDir: this.dataDir,
|
||||
logger: this.logger,
|
||||
scheduler: this.scheduler,
|
||||
originalUserText: turn.userText,
|
||||
originalUserMessageCount: turn.originalUserMessageCount,
|
||||
pluginStartTimestamp: turn.startedAt ?? Date.now(),
|
||||
vectorStore: this.vectorStore,
|
||||
embeddingService: this.embeddingService,
|
||||
bgTaskRegistry: this.bgTasks,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Search L1 structured memories.
|
||||
* Maps to: `tdai_memory_search` tool.
|
||||
*/
|
||||
async searchMemories(params: MemorySearchParams): Promise<{ text: string; total: number; strategy: string }> {
|
||||
const result = await executeMemorySearch({
|
||||
query: params.query,
|
||||
limit: params.limit ?? 5,
|
||||
type: params.type,
|
||||
scene: params.scene,
|
||||
vectorStore: this.vectorStore,
|
||||
embeddingService: this.embeddingService,
|
||||
logger: this.logger,
|
||||
});
|
||||
|
||||
return {
|
||||
text: formatSearchResponse(result),
|
||||
total: result.total,
|
||||
strategy: result.strategy,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Search L0 raw conversations.
|
||||
* Maps to: `tdai_conversation_search` tool.
|
||||
*/
|
||||
async searchConversations(params: ConversationSearchParams): Promise<{ text: string; total: number }> {
|
||||
const result = await executeConversationSearch({
|
||||
query: params.query,
|
||||
limit: params.limit ?? 5,
|
||||
sessionKey: params.sessionKey,
|
||||
vectorStore: this.vectorStore,
|
||||
embeddingService: this.embeddingService,
|
||||
logger: this.logger,
|
||||
});
|
||||
|
||||
return {
|
||||
text: formatConversationSearchResponse(result),
|
||||
total: result.total,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle end-of-conversation for a single session.
|
||||
*
|
||||
* ⚠️ Read this if you are editing the method:
|
||||
*
|
||||
* There are two distinct shutdown-ish events, and they must **NOT**
|
||||
* share an implementation:
|
||||
*
|
||||
* - **`gateway_stop` (OpenClaw / process exit)**
|
||||
* The host is going away. Tear everything down — scheduler,
|
||||
* VectorStore, EmbeddingService, caches. That is
|
||||
* {@link destroy}, not this method.
|
||||
*
|
||||
* - **`on_session_end` (Hermes) / `POST /session/end` (Gateway)**
|
||||
* One conversation ended while the process keeps serving other
|
||||
* concurrent sessions. **Only** this session's buffered work
|
||||
* should be flushed; every other session's timers, buffers,
|
||||
* pipeline state, and the shared scheduler itself MUST remain
|
||||
* untouched. That is this method.
|
||||
*
|
||||
* Historically this method did ``scheduler.destroy() +
|
||||
* createPipelineManager()``, which conflated the two semantics and
|
||||
* wiped concurrent sessions' in-memory state on every ``/session/end``
|
||||
* call. That bug is covered by the concurrency test
|
||||
* ``P0-1: handleSessionEnd must be scoped to its session``.
|
||||
*
|
||||
* @param sessionKey Session whose buffered work should be flushed.
|
||||
* Unknown keys are tolerated as a no-op so callers
|
||||
* don't have to pre-check whether the session was
|
||||
* already evicted or never produced a capture.
|
||||
*/
|
||||
async handleSessionEnd(sessionKey: string): Promise<void> {
|
||||
if (!sessionKey) return;
|
||||
await this.storeReady?.catch(() => {});
|
||||
if (!this.scheduler) return;
|
||||
await this.scheduler.flushSession(sessionKey);
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Accessors (for migration bridge)
|
||||
// ============================
|
||||
|
||||
/** Get the LLM runner factory (for creating host-neutral LLM runners). */
|
||||
getLLMRunnerFactory(): LLMRunnerFactory {
|
||||
return this.runnerFactory;
|
||||
}
|
||||
|
||||
/** Get the shared VectorStore (may be undefined if init failed). */
|
||||
getVectorStore(): IMemoryStore | undefined {
|
||||
return this.vectorStore;
|
||||
}
|
||||
|
||||
/** Get the shared EmbeddingService (may be undefined if not configured). */
|
||||
getEmbeddingService(): EmbeddingService | undefined {
|
||||
return this.embeddingService;
|
||||
}
|
||||
|
||||
/** Get the pipeline scheduler (may be undefined if extraction disabled). */
|
||||
getScheduler(): MemoryPipelineManager | undefined {
|
||||
return this.scheduler;
|
||||
}
|
||||
|
||||
/** Whether the scheduler has been started (or is currently starting). */
|
||||
isSchedulerStarted(): boolean {
|
||||
return this.schedulerStartPromise !== undefined;
|
||||
}
|
||||
|
||||
/** Set the instance ID for metrics (may be resolved asynchronously). */
|
||||
setInstanceId(id: string): void {
|
||||
this.instanceId = id;
|
||||
if (this.scheduler) {
|
||||
this.scheduler.instanceId = id;
|
||||
}
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Internal helpers
|
||||
// ============================
|
||||
|
||||
private async initStores(): Promise<void> {
|
||||
try {
|
||||
const stores = await initStores(this.cfg, this.dataDir, this.logger);
|
||||
this.vectorStore = stores.vectorStore;
|
||||
this.embeddingService = stores.embeddingService;
|
||||
this.logger.debug?.(`${TAG} Stores initialized: backend=${this.cfg.storeBackend}, embedding=${this.cfg.embedding.provider}`);
|
||||
} catch (err) {
|
||||
this.logger.warn(
|
||||
`${TAG} Store init failed; recall/dedup degraded: ${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private wirePipelineRunners(): void {
|
||||
if (!this.scheduler) return;
|
||||
|
||||
// Determine whether to use standalone LLM runner for extraction.
|
||||
// Priority: cfg.llm.enabled (explicit override) > hostType detection.
|
||||
const useStandaloneRunner = this.cfg.llm.enabled || this.hostAdapter.hostType !== "openclaw";
|
||||
|
||||
const openclawConfig = (!useStandaloneRunner && this.hostAdapter.hostType === "openclaw")
|
||||
? (this.hostAdapter as { getOpenClawConfig?(): unknown }).getOpenClawConfig?.()
|
||||
: undefined;
|
||||
|
||||
// When standalone runner is active, create LLM runners from the factory.
|
||||
// If cfg.llm is configured AND we're in OpenClaw mode, build a dedicated
|
||||
// StandaloneLLMRunnerFactory from cfg.llm to override the host runner.
|
||||
let runnerFactory = this.runnerFactory;
|
||||
if (useStandaloneRunner && this.cfg.llm.enabled && this.hostAdapter.hostType === "openclaw") {
|
||||
runnerFactory = new StandaloneLLMRunnerFactory({
|
||||
config: {
|
||||
baseUrl: this.cfg.llm.baseUrl,
|
||||
apiKey: this.cfg.llm.apiKey,
|
||||
model: this.cfg.llm.model,
|
||||
maxTokens: this.cfg.llm.maxTokens,
|
||||
timeoutMs: this.cfg.llm.timeoutMs,
|
||||
},
|
||||
logger: this.logger,
|
||||
});
|
||||
this.logger.debug?.(`${TAG} Using standalone LLM override: model=${this.cfg.llm.model}, baseUrl=${this.cfg.llm.baseUrl}`);
|
||||
}
|
||||
|
||||
const l1LlmRunner = useStandaloneRunner
|
||||
? runnerFactory.createRunner({ enableTools: false })
|
||||
: undefined;
|
||||
const l2l3LlmRunner = useStandaloneRunner
|
||||
? runnerFactory.createRunner({ enableTools: true })
|
||||
: undefined;
|
||||
|
||||
// L1 runner
|
||||
this.scheduler.setL1Runner(createL1Runner({
|
||||
pluginDataDir: this.dataDir,
|
||||
cfg: this.cfg,
|
||||
openclawConfig,
|
||||
vectorStore: this.vectorStore,
|
||||
embeddingService: this.embeddingService,
|
||||
logger: this.logger,
|
||||
getInstanceId: () => this.instanceId,
|
||||
llmRunner: l1LlmRunner,
|
||||
}));
|
||||
|
||||
// Persister
|
||||
this.scheduler.setPersister(createPersister(this.dataDir, this.logger));
|
||||
|
||||
// L2 runner
|
||||
this.scheduler.setL2Runner(async (sessionKey: string, cursor?: string) => {
|
||||
const l2Runner = createL2Runner({
|
||||
pluginDataDir: this.dataDir,
|
||||
cfg: this.cfg,
|
||||
openclawConfig,
|
||||
vectorStore: this.vectorStore,
|
||||
logger: this.logger,
|
||||
instanceId: this.instanceId,
|
||||
llmRunner: l2l3LlmRunner,
|
||||
});
|
||||
return l2Runner(sessionKey, cursor);
|
||||
});
|
||||
|
||||
// L3 runner
|
||||
this.scheduler.setL3Runner(async () => {
|
||||
const l3Runner = createL3Runner({
|
||||
pluginDataDir: this.dataDir,
|
||||
cfg: this.cfg,
|
||||
openclawConfig,
|
||||
vectorStore: this.vectorStore,
|
||||
logger: this.logger,
|
||||
instanceId: this.instanceId,
|
||||
llmRunner: l2l3LlmRunner,
|
||||
});
|
||||
await l3Runner();
|
||||
});
|
||||
|
||||
this.logger.debug?.(`${TAG} Pipeline runners wired`);
|
||||
}
|
||||
|
||||
private ensureSchedulerStarted(): Promise<void> {
|
||||
// Fast path: already started (or starting) — every concurrent caller
|
||||
// awaits the same in-flight promise. The promise is kept around as a
|
||||
// permanently-resolved sentinel after success so subsequent calls
|
||||
// collapse into a cheap already-resolved await.
|
||||
if (this.schedulerStartPromise) return this.schedulerStartPromise;
|
||||
if (!this.scheduler) return Promise.resolve();
|
||||
|
||||
// Capture scheduler locally so TypeScript narrows inside the closure
|
||||
// even after ``this.scheduler`` is re-assigned by handleSessionEnd.
|
||||
const scheduler = this.scheduler;
|
||||
this.schedulerStartPromise = (async () => {
|
||||
try {
|
||||
const checkpoint = new CheckpointManager(this.dataDir, this.logger);
|
||||
const cp = await checkpoint.read();
|
||||
scheduler.start(checkpoint.getAllPipelineStates(cp));
|
||||
this.logger.debug?.(`${TAG} Scheduler started`);
|
||||
} catch (err) {
|
||||
this.logger.error(`${TAG} Failed to restore checkpoint: ${err instanceof Error ? err.message : String(err)}`);
|
||||
scheduler.start({});
|
||||
}
|
||||
})();
|
||||
|
||||
// If the start sequence itself rejects we clear the gate so the next
|
||||
// caller can retry; on success we keep the resolved promise so it
|
||||
// short-circuits permanently.
|
||||
this.schedulerStartPromise.catch(() => {
|
||||
this.schedulerStartPromise = undefined;
|
||||
});
|
||||
|
||||
return this.schedulerStartPromise;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,279 @@
|
||||
/**
|
||||
* conversation_search tool: Agent-callable tool for searching L0 conversation records.
|
||||
*
|
||||
* Supports three search strategies with automatic degradation:
|
||||
* 1. **hybrid** (default) — FTS5 keyword + vector embedding in parallel,
|
||||
* merged via Reciprocal Rank Fusion (RRF).
|
||||
* 2. **embedding** — pure vector similarity (when FTS5 is unavailable).
|
||||
* 3. **fts** — pure FTS5 keyword search (when embedding is unavailable).
|
||||
*
|
||||
* The tool is registered via `api.registerTool()` in index.ts.
|
||||
*/
|
||||
|
||||
import type { IMemoryStore, L0SearchResult } from "../store/types.js";
|
||||
import { buildFtsQuery } from "../store/sqlite.js";
|
||||
import type { EmbeddingService } from "../store/embedding.js";
|
||||
|
||||
// ============================
|
||||
// Types
|
||||
// ============================
|
||||
|
||||
interface Logger {
|
||||
debug?: (message: string) => void;
|
||||
info: (message: string) => void;
|
||||
warn: (message: string) => void;
|
||||
error: (message: string) => void;
|
||||
}
|
||||
|
||||
export interface ConversationSearchResultItem {
|
||||
id: string;
|
||||
session_key: string;
|
||||
/** Role of the message sender: "user" or "assistant" */
|
||||
role: string;
|
||||
/** Text content of this single message */
|
||||
content: string;
|
||||
score: number;
|
||||
recorded_at: string;
|
||||
}
|
||||
|
||||
export interface ConversationSearchResult {
|
||||
results: ConversationSearchResultItem[];
|
||||
total: number;
|
||||
/** Actual search strategy used: "hybrid", "embedding", "fts", or "none". */
|
||||
strategy: string;
|
||||
/** Optional message, e.g. when embedding is not configured. */
|
||||
message?: string;
|
||||
}
|
||||
|
||||
const TAG = "[memory-tdai][tdai_conversation_search]";
|
||||
|
||||
// ============================
|
||||
// RRF (Reciprocal Rank Fusion)
|
||||
// ============================
|
||||
|
||||
/** Standard RRF constant from the original RRF paper. */
|
||||
const RRF_K = 60;
|
||||
|
||||
/**
|
||||
* Merge multiple ranked lists of `ConversationSearchResultItem` via Reciprocal
|
||||
* Rank Fusion. Items appearing in multiple lists get their RRF scores summed.
|
||||
*
|
||||
* Returns items sorted by descending RRF score. The `score` field of each
|
||||
* returned item is replaced by the RRF score for consistent ranking semantics.
|
||||
*/
|
||||
function rrfMergeL0(...lists: ConversationSearchResultItem[][]): ConversationSearchResultItem[] {
|
||||
const map = new Map<string, { item: ConversationSearchResultItem; rrfScore: number }>();
|
||||
|
||||
for (const list of lists) {
|
||||
for (let rank = 0; rank < list.length; rank++) {
|
||||
const item = list[rank];
|
||||
const score = 1 / (RRF_K + rank + 1);
|
||||
const existing = map.get(item.id);
|
||||
if (existing) {
|
||||
existing.rrfScore += score;
|
||||
} else {
|
||||
map.set(item.id, { item, rrfScore: score });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return [...map.values()]
|
||||
.sort((a, b) => b.rrfScore - a.rrfScore)
|
||||
.map(({ item, rrfScore }) => ({ ...item, score: rrfScore }));
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Search implementation
|
||||
// ============================
|
||||
|
||||
export async function executeConversationSearch(params: {
|
||||
query: string;
|
||||
limit: number;
|
||||
sessionKey?: string;
|
||||
vectorStore?: IMemoryStore;
|
||||
embeddingService?: EmbeddingService;
|
||||
logger?: Logger;
|
||||
}): Promise<ConversationSearchResult> {
|
||||
const {
|
||||
query,
|
||||
limit,
|
||||
sessionKey: sessionFilter,
|
||||
vectorStore,
|
||||
embeddingService,
|
||||
logger,
|
||||
} = params;
|
||||
|
||||
logger?.debug?.(
|
||||
`${TAG} CALLED: query="${query.slice(0, 100)}", limit=${limit}, ` +
|
||||
`sessionFilter=${sessionFilter ?? "(none)"}, ` +
|
||||
`vectorStore=${vectorStore ? "available" : "UNAVAILABLE"}, ` +
|
||||
`embeddingService=${embeddingService ? "available" : "UNAVAILABLE"}`,
|
||||
);
|
||||
|
||||
if (!query || query.trim().length === 0) {
|
||||
logger?.debug?.(`${TAG} Empty query, returning empty`);
|
||||
return { results: [], total: 0, strategy: "none" };
|
||||
}
|
||||
|
||||
if (!vectorStore) {
|
||||
logger?.warn?.(`${TAG} VectorStore not available`);
|
||||
return { results: [], total: 0, strategy: "none" };
|
||||
}
|
||||
|
||||
// ── Determine available capabilities ──
|
||||
const hasEmbedding = !!embeddingService;
|
||||
const hasFts = vectorStore.isFtsAvailable();
|
||||
|
||||
if (!hasEmbedding && !hasFts) {
|
||||
logger?.warn?.(`${TAG} Neither EmbeddingService nor FTS5 available — cannot search`);
|
||||
return {
|
||||
results: [],
|
||||
total: 0,
|
||||
strategy: "none",
|
||||
message:
|
||||
"Embedding service is not configured and FTS is not available. " +
|
||||
"Conversation search requires an embedding provider or FTS5 support. " +
|
||||
"Please configure an embedding provider in the embedding.provider setting (e.g. openai_compatible).",
|
||||
};
|
||||
}
|
||||
|
||||
// ── Over-retrieve for later filtering and RRF merging ──
|
||||
const candidateK = sessionFilter ? limit * 4 : limit * 3;
|
||||
|
||||
// ── Run available search strategies in parallel ──
|
||||
const [ftsItems, vecItems] = await Promise.all([
|
||||
// FTS5 keyword search on L0
|
||||
(async (): Promise<ConversationSearchResultItem[]> => {
|
||||
if (!hasFts) return [];
|
||||
try {
|
||||
const ftsQuery = buildFtsQuery(query);
|
||||
if (!ftsQuery) {
|
||||
logger?.debug?.(`${TAG} [hybrid-fts] No usable FTS tokens from query`);
|
||||
return [];
|
||||
}
|
||||
logger?.debug?.(`${TAG} [hybrid-fts] FTS5 query: "${ftsQuery}"`);
|
||||
const ftsResults = await vectorStore.searchL0Fts(ftsQuery, candidateK);
|
||||
logger?.debug?.(`${TAG} [hybrid-fts] FTS5 returned ${ftsResults.length} candidates`);
|
||||
return ftsResults.map((r) => ({
|
||||
id: r.record_id,
|
||||
session_key: r.session_key,
|
||||
role: r.role,
|
||||
content: r.message_text,
|
||||
score: r.score,
|
||||
recorded_at: r.recorded_at,
|
||||
}));
|
||||
} catch (err) {
|
||||
logger?.warn?.(
|
||||
`${TAG} [hybrid-fts] FTS5 search failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
return [];
|
||||
}
|
||||
})(),
|
||||
|
||||
// Vector embedding search on L0
|
||||
(async (): Promise<ConversationSearchResultItem[]> => {
|
||||
if (!hasEmbedding) return [];
|
||||
try {
|
||||
logger?.debug?.(`${TAG} [hybrid-vec] Generating query embedding...`);
|
||||
const queryEmbedding = await embeddingService!.embed(query);
|
||||
logger?.debug?.(
|
||||
`${TAG} [hybrid-vec] Embedding OK, dims=${queryEmbedding.length}, searching top-${candidateK}...`,
|
||||
);
|
||||
const vecResults: L0SearchResult[] = await vectorStore.searchL0Vector(queryEmbedding, candidateK, query);
|
||||
logger?.debug?.(`${TAG} [hybrid-vec] Vector search returned ${vecResults.length} candidates`);
|
||||
return vecResults.map((r) => ({
|
||||
id: r.record_id,
|
||||
session_key: r.session_key,
|
||||
role: r.role,
|
||||
content: r.message_text,
|
||||
score: r.score,
|
||||
recorded_at: r.recorded_at,
|
||||
}));
|
||||
} catch (err) {
|
||||
logger?.warn?.(
|
||||
`${TAG} [hybrid-vec] Embedding search failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
return [];
|
||||
}
|
||||
})(),
|
||||
]);
|
||||
|
||||
// ── Determine effective strategy ──
|
||||
const ftsOk = ftsItems.length > 0;
|
||||
const vecOk = vecItems.length > 0;
|
||||
let strategy: string;
|
||||
|
||||
if (ftsOk && vecOk) {
|
||||
strategy = "hybrid";
|
||||
} else if (vecOk) {
|
||||
strategy = "embedding";
|
||||
} else if (ftsOk) {
|
||||
strategy = "fts";
|
||||
} else {
|
||||
logger?.debug?.(`${TAG} Both search paths returned 0 results`);
|
||||
return { results: [], total: 0, strategy: hasEmbedding ? "embedding" : "fts" };
|
||||
}
|
||||
|
||||
// ── Merge results ──
|
||||
let results: ConversationSearchResultItem[];
|
||||
if (strategy === "hybrid") {
|
||||
results = rrfMergeL0(ftsItems, vecItems);
|
||||
logger?.debug?.(
|
||||
`${TAG} [hybrid] RRF merged: fts=${ftsItems.length}, vec=${vecItems.length} → ${results.length} unique`,
|
||||
);
|
||||
} else {
|
||||
// Single-source: use whichever list has results (already sorted by score)
|
||||
results = ftsOk ? ftsItems : vecItems;
|
||||
}
|
||||
|
||||
// ── Apply session key filter ──
|
||||
if (sessionFilter) {
|
||||
const preFilterCount = results.length;
|
||||
results = results.filter((r) => r.session_key === sessionFilter);
|
||||
logger?.debug?.(`${TAG} After session filter "${sessionFilter}": ${results.length}/${preFilterCount}`);
|
||||
}
|
||||
|
||||
// ── Trim to requested limit ──
|
||||
const trimmed = results.slice(0, limit);
|
||||
|
||||
logger?.debug?.(
|
||||
`${TAG} RESULT (strategy=${strategy}): returning ${trimmed.length} messages ` +
|
||||
`(scores: [${trimmed.map((r) => r.score.toFixed(3)).join(", ")}])`,
|
||||
);
|
||||
|
||||
return {
|
||||
results: trimmed,
|
||||
total: trimmed.length,
|
||||
strategy,
|
||||
};
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Tool response formatter
|
||||
// ============================
|
||||
|
||||
export function formatConversationSearchResponse(result: ConversationSearchResult): string {
|
||||
if (result.message) {
|
||||
return result.message;
|
||||
}
|
||||
if (result.results.length === 0) {
|
||||
return "No matching conversation messages found.";
|
||||
}
|
||||
|
||||
const lines: string[] = [
|
||||
`Found ${result.total} matching message(s):`,
|
||||
"",
|
||||
];
|
||||
|
||||
for (const item of result.results) {
|
||||
const scoreStr = typeof item.score === "number" ? ` (score: ${item.score.toFixed(3)})` : "";
|
||||
const dateStr = item.recorded_at ? ` [${item.recorded_at}]` : "";
|
||||
lines.push(`---`);
|
||||
lines.push(`**[${item.role}]** Session: ${item.session_key}${dateStr}${scoreStr}`);
|
||||
lines.push("");
|
||||
lines.push(item.content);
|
||||
lines.push("");
|
||||
}
|
||||
|
||||
return lines.join("\n");
|
||||
}
|
||||
@@ -0,0 +1,290 @@
|
||||
/**
|
||||
* memory_search tool: Agent-callable tool for searching L1 memory records.
|
||||
*
|
||||
* Supports three search strategies with automatic degradation:
|
||||
* 1. **hybrid** (default) — FTS5 keyword + vector embedding in parallel,
|
||||
* merged via Reciprocal Rank Fusion (RRF).
|
||||
* 2. **embedding** — pure vector similarity (when FTS5 is unavailable).
|
||||
* 3. **fts** — pure FTS5 keyword search (when embedding is unavailable).
|
||||
*
|
||||
* The tool is registered via `api.registerTool()` in index.ts.
|
||||
*/
|
||||
|
||||
import type { IMemoryStore, L1SearchResult } from "../store/types.js";
|
||||
import { buildFtsQuery } from "../store/sqlite.js";
|
||||
import type { EmbeddingService } from "../store/embedding.js";
|
||||
|
||||
// ============================
|
||||
// Types
|
||||
// ============================
|
||||
|
||||
interface Logger {
|
||||
debug?: (message: string) => void;
|
||||
info: (message: string) => void;
|
||||
warn: (message: string) => void;
|
||||
error: (message: string) => void;
|
||||
}
|
||||
|
||||
export interface MemorySearchResultItem {
|
||||
id: string;
|
||||
content: string;
|
||||
type: string;
|
||||
priority: number;
|
||||
scene_name: string;
|
||||
score: number;
|
||||
created_at: string;
|
||||
updated_at: string;
|
||||
}
|
||||
|
||||
export interface MemorySearchResult {
|
||||
results: MemorySearchResultItem[];
|
||||
total: number;
|
||||
strategy: string;
|
||||
/** Optional message, e.g. when embedding is not configured. */
|
||||
message?: string;
|
||||
}
|
||||
|
||||
const TAG = "[memory-tdai][tdai_memory_search]";
|
||||
|
||||
// ============================
|
||||
// RRF (Reciprocal Rank Fusion)
|
||||
// ============================
|
||||
|
||||
/** Standard RRF constant from the original RRF paper. */
|
||||
const RRF_K = 60;
|
||||
|
||||
/**
|
||||
* Merge multiple ranked lists of `MemorySearchResultItem` via Reciprocal Rank
|
||||
* Fusion. Items appearing in multiple lists get their RRF scores summed.
|
||||
*
|
||||
* Returns items sorted by descending RRF score. The `score` field of each
|
||||
* returned item is replaced by the RRF score for consistent ranking semantics.
|
||||
*/
|
||||
function rrfMergeL1(...lists: MemorySearchResultItem[][]): MemorySearchResultItem[] {
|
||||
const map = new Map<string, { item: MemorySearchResultItem; rrfScore: number }>();
|
||||
|
||||
for (const list of lists) {
|
||||
for (let rank = 0; rank < list.length; rank++) {
|
||||
const item = list[rank];
|
||||
const score = 1 / (RRF_K + rank + 1);
|
||||
const existing = map.get(item.id);
|
||||
if (existing) {
|
||||
existing.rrfScore += score;
|
||||
} else {
|
||||
map.set(item.id, { item, rrfScore: score });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return [...map.values()]
|
||||
.sort((a, b) => b.rrfScore - a.rrfScore)
|
||||
.map(({ item, rrfScore }) => ({ ...item, score: rrfScore }));
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Search implementation
|
||||
// ============================
|
||||
|
||||
export async function executeMemorySearch(params: {
|
||||
query: string;
|
||||
limit: number;
|
||||
type?: string;
|
||||
scene?: string;
|
||||
vectorStore?: IMemoryStore;
|
||||
embeddingService?: EmbeddingService;
|
||||
logger?: Logger;
|
||||
}): Promise<MemorySearchResult> {
|
||||
const {
|
||||
query,
|
||||
limit,
|
||||
type: typeFilter,
|
||||
scene: sceneFilter,
|
||||
vectorStore,
|
||||
embeddingService,
|
||||
logger,
|
||||
} = params;
|
||||
|
||||
logger?.debug?.(
|
||||
`${TAG} CALLED: query="${query.slice(0, 100)}", limit=${limit}, ` +
|
||||
`typeFilter=${typeFilter ?? "(none)"}, sceneFilter=${sceneFilter ?? "(none)"}, ` +
|
||||
`vectorStore=${vectorStore ? "available" : "UNAVAILABLE"}, ` +
|
||||
`embeddingService=${embeddingService ? "available" : "UNAVAILABLE"}`,
|
||||
);
|
||||
|
||||
if (!query || query.trim().length === 0) {
|
||||
logger?.debug?.(`${TAG} Empty query, returning empty`);
|
||||
return { results: [], total: 0, strategy: "none" };
|
||||
}
|
||||
|
||||
if (!vectorStore) {
|
||||
logger?.warn?.(`${TAG} VectorStore not available`);
|
||||
return { results: [], total: 0, strategy: "none" };
|
||||
}
|
||||
|
||||
// ── Determine available capabilities ──
|
||||
const hasEmbedding = !!embeddingService;
|
||||
const hasFts = vectorStore.isFtsAvailable();
|
||||
|
||||
if (!hasEmbedding && !hasFts) {
|
||||
logger?.warn?.(`${TAG} Neither EmbeddingService nor FTS5 available — cannot search`);
|
||||
return {
|
||||
results: [],
|
||||
total: 0,
|
||||
strategy: "none",
|
||||
message:
|
||||
"Embedding service is not configured and FTS is not available. " +
|
||||
"Memory search requires an embedding provider or FTS5 support. " +
|
||||
"Please configure an embedding provider in the embedding.provider setting (e.g. openai_compatible).",
|
||||
};
|
||||
}
|
||||
|
||||
// ── Over-retrieve for later filtering and RRF merging ──
|
||||
const candidateK = limit * 3;
|
||||
|
||||
// ── Run available search strategies in parallel ──
|
||||
const [ftsItems, vecItems] = await Promise.all([
|
||||
// FTS5 keyword search
|
||||
(async (): Promise<MemorySearchResultItem[]> => {
|
||||
if (!hasFts) return [];
|
||||
try {
|
||||
const ftsQuery = buildFtsQuery(query);
|
||||
if (!ftsQuery) {
|
||||
logger?.debug?.(`${TAG} [hybrid-fts] No usable FTS tokens from query`);
|
||||
return [];
|
||||
}
|
||||
logger?.debug?.(`${TAG} [hybrid-fts] FTS5 query: "${ftsQuery}"`);
|
||||
const ftsResults = await vectorStore.searchL1Fts(ftsQuery, candidateK);
|
||||
logger?.debug?.(`${TAG} [hybrid-fts] FTS5 returned ${ftsResults.length} candidates`);
|
||||
return ftsResults.map((r) => ({
|
||||
id: r.record_id,
|
||||
content: r.content,
|
||||
type: r.type,
|
||||
priority: r.priority,
|
||||
scene_name: r.scene_name,
|
||||
score: r.score,
|
||||
created_at: r.timestamp_start,
|
||||
updated_at: r.timestamp_end,
|
||||
}));
|
||||
} catch (err) {
|
||||
logger?.warn?.(
|
||||
`${TAG} [hybrid-fts] FTS5 search failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
return [];
|
||||
}
|
||||
})(),
|
||||
|
||||
// Vector embedding search
|
||||
(async (): Promise<MemorySearchResultItem[]> => {
|
||||
if (!hasEmbedding) return [];
|
||||
try {
|
||||
logger?.debug?.(`${TAG} [hybrid-vec] Generating query embedding...`);
|
||||
const queryEmbedding = await embeddingService!.embed(query);
|
||||
logger?.debug?.(
|
||||
`${TAG} [hybrid-vec] Embedding OK, dims=${queryEmbedding.length}, searching top-${candidateK}...`,
|
||||
);
|
||||
const vecResults: L1SearchResult[] = await vectorStore.searchL1Vector(queryEmbedding, candidateK, query);
|
||||
logger?.debug?.(`${TAG} [hybrid-vec] Vector search returned ${vecResults.length} candidates`);
|
||||
return vecResults.map((r) => ({
|
||||
id: r.record_id,
|
||||
content: r.content,
|
||||
type: r.type,
|
||||
priority: r.priority,
|
||||
scene_name: r.scene_name,
|
||||
score: r.score,
|
||||
created_at: r.timestamp_start,
|
||||
updated_at: r.timestamp_end,
|
||||
}));
|
||||
} catch (err) {
|
||||
logger?.warn?.(
|
||||
`${TAG} [hybrid-vec] Embedding search failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
return [];
|
||||
}
|
||||
})(),
|
||||
]);
|
||||
|
||||
// ── Determine effective strategy ──
|
||||
const ftsOk = ftsItems.length > 0;
|
||||
const vecOk = vecItems.length > 0;
|
||||
let strategy: string;
|
||||
|
||||
if (ftsOk && vecOk) {
|
||||
strategy = "hybrid";
|
||||
} else if (vecOk) {
|
||||
strategy = "embedding";
|
||||
} else if (ftsOk) {
|
||||
strategy = "fts";
|
||||
} else {
|
||||
logger?.debug?.(`${TAG} Both search paths returned 0 results`);
|
||||
return { results: [], total: 0, strategy: hasEmbedding ? "embedding" : "fts" };
|
||||
}
|
||||
|
||||
// ── Merge results ──
|
||||
let results: MemorySearchResultItem[];
|
||||
if (strategy === "hybrid") {
|
||||
results = rrfMergeL1(ftsItems, vecItems);
|
||||
logger?.debug?.(
|
||||
`${TAG} [hybrid] RRF merged: fts=${ftsItems.length}, vec=${vecItems.length} → ${results.length} unique`,
|
||||
);
|
||||
} else {
|
||||
// Single-source: use whichever list has results (already sorted by score)
|
||||
results = ftsOk ? ftsItems : vecItems;
|
||||
}
|
||||
|
||||
// ── Apply secondary filters (type, scene) ──
|
||||
const preFilterCount = results.length;
|
||||
if (typeFilter) {
|
||||
results = results.filter((r) => r.type === typeFilter);
|
||||
logger?.debug?.(`${TAG} After type filter "${typeFilter}": ${results.length}/${preFilterCount}`);
|
||||
}
|
||||
if (sceneFilter) {
|
||||
const normalizedScene = sceneFilter.toLowerCase();
|
||||
results = results.filter((r) =>
|
||||
r.scene_name.toLowerCase().includes(normalizedScene),
|
||||
);
|
||||
logger?.debug?.(`${TAG} After scene filter "${sceneFilter}": ${results.length}/${preFilterCount}`);
|
||||
}
|
||||
|
||||
// ── Trim to requested limit ──
|
||||
const trimmed = results.slice(0, limit);
|
||||
|
||||
logger?.debug?.(
|
||||
`${TAG} RESULT (strategy=${strategy}): returning ${trimmed.length} memories ` +
|
||||
`(scores: [${trimmed.map((r) => r.score.toFixed(3)).join(", ")}])`,
|
||||
);
|
||||
|
||||
return {
|
||||
results: trimmed,
|
||||
total: trimmed.length,
|
||||
strategy,
|
||||
};
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Tool response formatter
|
||||
// ============================
|
||||
|
||||
export function formatSearchResponse(result: MemorySearchResult): string {
|
||||
if (result.message) {
|
||||
return result.message;
|
||||
}
|
||||
if (result.results.length === 0) {
|
||||
return "No matching memories found.";
|
||||
}
|
||||
|
||||
const lines: string[] = [
|
||||
`Found ${result.total} matching memories:`,
|
||||
"",
|
||||
];
|
||||
|
||||
for (const item of result.results) {
|
||||
const scoreStr = typeof item.score === "number" ? ` (score: ${item.score.toFixed(3)})` : "";
|
||||
const sceneStr = item.scene_name ? ` [scene: ${item.scene_name}]` : "";
|
||||
const priorityStr = item.priority >= 0 ? ` (priority: ${item.priority})` : " (global instruction)";
|
||||
lines.push(`- **[${item.type}]**${priorityStr}${sceneStr}${scoreStr}`);
|
||||
lines.push(` ${item.content}`);
|
||||
lines.push("");
|
||||
}
|
||||
|
||||
return lines.join("\n");
|
||||
}
|
||||
@@ -0,0 +1,241 @@
|
||||
/**
|
||||
* TDAI Core — Host-neutral type definitions and abstract interfaces.
|
||||
*
|
||||
* These types define the boundary between TDAI Core (memory algorithms)
|
||||
* and the host environment (OpenClaw, Hermes, standalone Gateway, etc.).
|
||||
*
|
||||
* Design principles:
|
||||
* 1. TDAI Core depends ONLY on these interfaces — never on a specific host.
|
||||
* 2. Each host provides its own implementation of HostAdapter + LLMRunnerFactory.
|
||||
* 3. RuntimeContext is the single source of truth for session/user identity.
|
||||
*/
|
||||
|
||||
// ============================
|
||||
// Logger (unified across all layers)
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Minimal logger interface used throughout TDAI Core.
|
||||
*
|
||||
* Matches the existing `StoreLogger` and `RunnerLogger` interfaces
|
||||
* already used in the codebase — no migration needed for existing callers.
|
||||
*/
|
||||
export interface Logger {
|
||||
debug?: (message: string) => void;
|
||||
info: (message: string) => void;
|
||||
warn: (message: string) => void;
|
||||
error: (message: string) => void;
|
||||
}
|
||||
|
||||
// ============================
|
||||
// RuntimeContext
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Unified runtime context — provides identity, scoping, and path information.
|
||||
*
|
||||
* In OpenClaw: populated from `pluginConfig`, `sessionKey`, `resolveStateDir()`.
|
||||
* In Hermes: populated from `MemoryProvider.initialize()` kwargs.
|
||||
* In Gateway: populated from HTTP request parameters.
|
||||
*/
|
||||
export interface RuntimeContext {
|
||||
/** User identifier (e.g. "default_user" for CLI, platform user ID for gateway). */
|
||||
userId: string;
|
||||
/** Session identifier (unique per conversation session). */
|
||||
sessionId: string;
|
||||
/** Session key (stable across reconnects, used for L0/L1 grouping). */
|
||||
sessionKey: string;
|
||||
/** Host platform identifier. */
|
||||
platform: "openclaw" | "hermes" | "cli" | "gateway" | string;
|
||||
/** Agent identity / profile name (optional). */
|
||||
agentIdentity?: string;
|
||||
/** Agent execution context — primary agent, subagent, cron job, or flush task. */
|
||||
agentContext?: "primary" | "subagent" | "cron" | "flush";
|
||||
/** Workspace directory (for tool sandbox, if applicable). */
|
||||
workspaceDir: string;
|
||||
/** Plugin/provider data directory (L0, records, scene_blocks, etc.). */
|
||||
dataDir: string;
|
||||
}
|
||||
|
||||
// ============================
|
||||
// LLMRunner
|
||||
// ============================
|
||||
|
||||
/** Parameters for a single LLM execution. */
|
||||
export interface LLMRunParams {
|
||||
/** User-facing prompt (or combined prompt if no systemPrompt). */
|
||||
prompt: string;
|
||||
/** Optional system prompt. When provided, `prompt` is used as the user message. */
|
||||
systemPrompt?: string;
|
||||
/** Unique task identifier for logging and metrics. */
|
||||
taskId: string;
|
||||
/** Execution timeout in milliseconds (default: 120_000). */
|
||||
timeoutMs?: number;
|
||||
/** Max output tokens (optional — defaults to model catalog value). */
|
||||
maxTokens?: number;
|
||||
/**
|
||||
* Working directory for tool-enabled runs.
|
||||
* When `enableTools` is true, the LLM's file tools resolve paths relative to this dir.
|
||||
* When omitted, a clean empty workspace is used.
|
||||
*/
|
||||
workspaceDir?: string;
|
||||
/** Plugin instance ID for metric reporting (optional). */
|
||||
instanceId?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Unified LLM execution interface.
|
||||
*
|
||||
* Replaces direct usage of `CleanContextRunner` throughout TDAI Core.
|
||||
*
|
||||
* Implementations:
|
||||
* - `OpenClawLLMRunner`: wraps `CleanContextRunner` / `runEmbeddedPiAgent` (OpenClaw host)
|
||||
* - `StandaloneLLMRunner`: direct OpenAI-compatible HTTP calls (Gateway / Hermes host)
|
||||
*/
|
||||
export interface LLMRunner {
|
||||
/**
|
||||
* Execute a prompt and return the LLM's text output.
|
||||
*
|
||||
* Behavior depends on the factory configuration:
|
||||
* - `enableTools: false` → pure text output (used by L1 extraction, L1 dedup)
|
||||
* - `enableTools: true` → LLM may call file tools (used by L2 scene, L3 persona)
|
||||
*
|
||||
* @returns The LLM's text response. Empty string if the LLM produces no output.
|
||||
* @throws On timeout, network errors, or unrecoverable LLM failures.
|
||||
*/
|
||||
run(params: LLMRunParams): Promise<string>;
|
||||
}
|
||||
|
||||
// ============================
|
||||
// LLMRunnerFactory
|
||||
// ============================
|
||||
|
||||
/** Options for creating an LLMRunner instance. */
|
||||
export interface LLMRunnerCreateOptions {
|
||||
/**
|
||||
* Full "provider/model" string (e.g. "openai/gpt-4o").
|
||||
* Takes precedence over host default model.
|
||||
*/
|
||||
modelRef?: string;
|
||||
/**
|
||||
* Whether the runner should allow tool calls (read_file, write_to_file, etc.).
|
||||
* Default: false (text-only output).
|
||||
*/
|
||||
enableTools?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Factory for creating LLMRunner instances.
|
||||
*
|
||||
* Each host provides its own factory implementation that knows how to
|
||||
* configure runners with the correct model, API keys, and tool sandbox.
|
||||
*/
|
||||
export interface LLMRunnerFactory {
|
||||
createRunner(opts?: LLMRunnerCreateOptions): LLMRunner;
|
||||
}
|
||||
|
||||
// ============================
|
||||
// HostAdapter
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Host adapter — translates host-specific events, context, and capabilities
|
||||
* into TDAI Core's unified interface.
|
||||
*
|
||||
* Each host environment provides exactly one HostAdapter implementation:
|
||||
* - OpenClaw: `OpenClawHostAdapter` — wraps `OpenClawPluginApi`
|
||||
* - Hermes/GW: `StandaloneHostAdapter` — wraps Gateway HTTP request context
|
||||
*
|
||||
* HostAdapter answers these questions for TDAI Core:
|
||||
* - "Who is the current user/session?" → `getRuntimeContext()`
|
||||
* - "How do I call an LLM?" → `getLLMRunnerFactory()`
|
||||
* - "Where do I log?" → `getLogger()`
|
||||
*/
|
||||
export interface HostAdapter {
|
||||
/** Identifies the host type for conditional behavior (should be rare). */
|
||||
readonly hostType: "openclaw" | "hermes" | "standalone";
|
||||
|
||||
/** Get the unified runtime context for the current session. */
|
||||
getRuntimeContext(): RuntimeContext;
|
||||
|
||||
/** Get the logger instance provided by the host. */
|
||||
getLogger(): Logger;
|
||||
|
||||
/** Get the LLM runner factory configured for this host. */
|
||||
getLLMRunnerFactory(): LLMRunnerFactory;
|
||||
}
|
||||
|
||||
// ============================
|
||||
// CompletedTurn — represents a finished conversation turn
|
||||
// ============================
|
||||
|
||||
/** A completed conversation turn, ready for capture/storage. */
|
||||
export interface CompletedTurn {
|
||||
/** The user's original message text. */
|
||||
userText: string;
|
||||
/** The assistant's response text. */
|
||||
assistantText: string;
|
||||
/** All messages in the turn (may include tool call results, etc.). */
|
||||
messages: unknown[];
|
||||
/** Session key for this turn. */
|
||||
sessionKey: string;
|
||||
/** Session ID within the session key (optional, for sub-session grouping). */
|
||||
sessionId?: string;
|
||||
/** Epoch ms when this turn started. */
|
||||
startedAt?: number;
|
||||
/**
|
||||
* Number of messages in the session at before_prompt_build time.
|
||||
* Used by l0-recorder to locate the exact user message that was
|
||||
* polluted by prependContext injection.
|
||||
*/
|
||||
originalUserMessageCount?: number;
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Core service result types
|
||||
// ============================
|
||||
|
||||
/** Result from a recall (prefetch) operation. */
|
||||
export interface RecallResult {
|
||||
/** L1 relevant memories — prepended to user prompt text (dynamic, per-turn). */
|
||||
prependContext?: string;
|
||||
/** Stable recall context appended to system prompt (persona, scene nav, tools guide). */
|
||||
appendSystemContext?: string;
|
||||
/** Recalled L1 memories with scores (for metrics). */
|
||||
recalledL1Memories?: Array<{ content: string; score: number; type: string }>;
|
||||
/** L3 Persona content (for metrics). */
|
||||
recalledL3Persona?: string | null;
|
||||
/** Search strategy used. */
|
||||
recallStrategy?: string;
|
||||
}
|
||||
|
||||
/** Result from a capture (sync_turn) operation. */
|
||||
export interface CaptureResult {
|
||||
/** Number of L0 messages recorded. */
|
||||
l0RecordedCount: number;
|
||||
/** Whether the pipeline scheduler was notified. */
|
||||
schedulerNotified: boolean;
|
||||
/** Number of L0 vectors written. */
|
||||
l0VectorsWritten: number;
|
||||
/** Filtered messages that were captured. */
|
||||
filteredMessages: Array<{
|
||||
role: string;
|
||||
content: string;
|
||||
timestamp: number;
|
||||
}>;
|
||||
}
|
||||
|
||||
/** Search parameters for L1 memory search. */
|
||||
export interface MemorySearchParams {
|
||||
query: string;
|
||||
limit?: number;
|
||||
type?: string;
|
||||
scene?: string;
|
||||
}
|
||||
|
||||
/** Search parameters for L0 conversation search. */
|
||||
export interface ConversationSearchParams {
|
||||
query: string;
|
||||
limit?: number;
|
||||
sessionKey?: string;
|
||||
}
|
||||
Reference in New Issue
Block a user