mirror of
https://github.com/TencentCloud/TencentDB-Agent-Memory
synced 2026-07-10 12:34:27 +00:00
feat: init Agent-Memory
This commit is contained in:
chrishuan
+419
@@ -0,0 +1,419 @@
|
||||
/**
|
||||
* Plugin configuration types and parser (v3).
|
||||
*
|
||||
* Config is organized into flat functional groups:
|
||||
* capture, extraction, persona, pipeline, recall, embedding
|
||||
*
|
||||
* Minimal config (zero config): {} — all fields have sensible defaults.
|
||||
*/
|
||||
|
||||
// ============================
|
||||
// Type definitions
|
||||
// ============================
|
||||
|
||||
/** Capture settings — controls L0 conversation recording. */
|
||||
export interface CaptureConfig {
|
||||
/** Enable auto-capture (default: true) */
|
||||
enabled: boolean;
|
||||
/** Glob patterns to exclude agents (e.g. "bench-judge-*"); matched agents are fully ignored */
|
||||
excludeAgents: string[];
|
||||
/**
|
||||
* L0/L1 local file retention days used as TTL switch.
|
||||
* 0 means cleanup disabled.(default: 0)
|
||||
*/
|
||||
l0l1RetentionDays: number;
|
||||
|
||||
/**
|
||||
* Allow dangerous low retention (1 or 2 days).
|
||||
* Default false: when disabled, non-zero retention must be >= 3.
|
||||
*/
|
||||
allowAggressiveCleanup: boolean;
|
||||
}
|
||||
|
||||
/** Extraction settings (L1) — controls memory extraction from conversations. */
|
||||
export interface ExtractionConfig {
|
||||
/** Enable background extraction (default: true) */
|
||||
enabled: boolean;
|
||||
/** Enable L1 smart dedup (default: true) */
|
||||
enableDedup: boolean;
|
||||
/** Max memories per session (default: 20) */
|
||||
maxMemoriesPerSession: number;
|
||||
/** LLM model for extraction, format: "provider/model" (falls back to OpenClaw default model when omitted) */
|
||||
model?: string;
|
||||
}
|
||||
|
||||
/** Persona (L2/L3) settings — controls scene extraction (L2) and user profile generation (L3). */
|
||||
export interface PersonaConfig {
|
||||
/** Trigger persona generation every N new memories (default: 50) */
|
||||
triggerEveryN: number;
|
||||
/** Max scene blocks (default: 20) */
|
||||
maxScenes: number;
|
||||
/** Persona backup count (default: 3) */
|
||||
backupCount: number;
|
||||
/** Scene blocks backup count (default: 10) */
|
||||
sceneBackupCount: number;
|
||||
/** LLM model for persona generation, format: "provider/model" (falls back to OpenClaw default model when omitted) */
|
||||
model?: string;
|
||||
}
|
||||
|
||||
/** Pipeline trigger settings (L1→L2→L3 scheduling). */
|
||||
export interface PipelineTriggerConfig {
|
||||
/** Trigger L1 after every N conversation rounds (default: 5) */
|
||||
everyNConversations: number;
|
||||
/** Enable warm-up: start threshold at 1, double after each L1 (1→2→4→...→everyN) (default: true) */
|
||||
enableWarmup: boolean;
|
||||
/** L1 idle timeout: trigger L1 after this many seconds of inactivity (default: 60) */
|
||||
l1IdleTimeoutSeconds: number;
|
||||
/** L2 delay after L1: wait this many seconds after L1 completes before triggering L2 (default: 90) */
|
||||
l2DelayAfterL1Seconds: number;
|
||||
/** L2 min interval: minimum seconds between L2 runs per session (default: 300 = 5 min) */
|
||||
l2MinIntervalSeconds: number;
|
||||
/** L2 max interval: even without new conversations, trigger L2 at most this often per session (default: 1800 = 30 min) */
|
||||
l2MaxIntervalSeconds: number;
|
||||
/** Sessions inactive longer than this (hours) stop L2 polling (default: 24) */
|
||||
sessionActiveWindowHours: number;
|
||||
}
|
||||
|
||||
/** Recall settings — controls memory retrieval for context injection. */
|
||||
export interface RecallConfig {
|
||||
/** Enable auto-recall (default: true) */
|
||||
enabled: boolean;
|
||||
/** Max results to return (default: 5) */
|
||||
maxResults: number;
|
||||
/** Minimum score threshold (default: 0.3) */
|
||||
scoreThreshold: number;
|
||||
/** Search strategy (default: "hybrid") */
|
||||
strategy: "embedding" | "keyword" | "hybrid";
|
||||
/** Overall recall timeout in milliseconds (default: 5000). When exceeded, recall is skipped with a warning. */
|
||||
timeoutMs: number;
|
||||
}
|
||||
|
||||
/** Embedding service configuration for vector search. */
|
||||
export interface EmbeddingConfig {
|
||||
/** User-facing default is true in schema, but provider="none" still disables embedding effectively. */
|
||||
enabled: boolean;
|
||||
/** Embedding provider: default "none" disables vector search; other values (e.g. "openai", "deepseek") are treated as OpenAI-compatible remote providers. */
|
||||
provider: string;
|
||||
/** API Base URL (required for remote provider). */
|
||||
baseUrl: string;
|
||||
/** API Key (required for remote provider). */
|
||||
apiKey: string;
|
||||
/** Model name (required for remote provider). */
|
||||
model: string;
|
||||
/** Vector dimensions (required for remote provider, must match model). */
|
||||
dimensions: number;
|
||||
/** Top-K candidates to recall during conflict detection (default: 5) */
|
||||
conflictRecallTopK: number;
|
||||
/** Proxy URL for qclaw provider — when provider="qclaw", requests are forwarded through this local proxy */
|
||||
proxyUrl?: string;
|
||||
/** Max input text length in characters before truncation (default: 5000). Texts exceeding this limit are truncated with a warning. */
|
||||
maxInputChars: number;
|
||||
/** Timeout per embedding API call in milliseconds (default: 10000). */
|
||||
timeoutMs: number;
|
||||
/** Internal-only local model cache directory, not exposed in plugin schema. */
|
||||
modelCacheDir?: string;
|
||||
/** If set, contains an error message about invalid remote config (embedding is disabled) */
|
||||
configError?: string;
|
||||
}
|
||||
|
||||
/** Daily cleaner settings for local JSONL data (L0/L1). */
|
||||
export interface MemoryCleanupConfig {
|
||||
/** TTL switch from capture.l0l1RetentionDays. Undefined means disabled. */
|
||||
retentionDays?: number;
|
||||
|
||||
/** Whether cleanup is enabled. True only when retentionDays is a valid positive number. */
|
||||
enabled: boolean;
|
||||
/** Daily execution time in HH:mm format (default: 03:00). */
|
||||
cleanTime: string;
|
||||
}
|
||||
|
||||
/** Report settings — controls metric/event reporting. */
|
||||
export interface ReportConfig {
|
||||
/** Enable reporting (default: true) */
|
||||
enabled: boolean;
|
||||
/** Reporter type: "local" logs structured JSON via logger (default: "local") */
|
||||
type: string;
|
||||
}
|
||||
|
||||
/** Fully resolved plugin configuration (v3). */
|
||||
export interface MemoryTdaiConfig {
|
||||
capture: CaptureConfig;
|
||||
extraction: ExtractionConfig;
|
||||
persona: PersonaConfig;
|
||||
pipeline: PipelineTriggerConfig;
|
||||
recall: RecallConfig;
|
||||
embedding: EmbeddingConfig;
|
||||
memoryCleanup: MemoryCleanupConfig;
|
||||
report: ReportConfig;
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Parser
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Parse plugin config from raw user input.
|
||||
* All fields have sensible defaults — minimal config is just {}.
|
||||
*/
|
||||
export function parseConfig(raw: Record<string, unknown> | undefined): MemoryTdaiConfig {
|
||||
const c = raw ?? {};
|
||||
|
||||
// --- Capture (L0) ---
|
||||
const captureGroup = obj(c, "capture");
|
||||
|
||||
// --- Retention days validation (from capture.l0l1RetentionDays) ---
|
||||
const rawRetentionDays = num(captureGroup, "l0l1RetentionDays") ?? 0;
|
||||
const allowAggressiveCleanup = bool(captureGroup, "allowAggressiveCleanup") ?? false;
|
||||
|
||||
let retentionDays: number | undefined;
|
||||
if (rawRetentionDays <= 0) {
|
||||
retentionDays = undefined;
|
||||
} else if (rawRetentionDays >= 3) {
|
||||
retentionDays = rawRetentionDays;
|
||||
} else if (allowAggressiveCleanup) {
|
||||
retentionDays = rawRetentionDays;
|
||||
} else {
|
||||
retentionDays = undefined;
|
||||
}
|
||||
|
||||
// --- Extraction (L1) ---
|
||||
const extractionGroup = obj(c, "extraction");
|
||||
|
||||
// --- Persona (L2/L3) ---
|
||||
const personaGroup = obj(c, "persona");
|
||||
|
||||
// --- Pipeline ---
|
||||
const pipelineGroup = obj(c, "pipeline");
|
||||
|
||||
// --- Recall ---
|
||||
const recallGroup = obj(c, "recall");
|
||||
|
||||
// --- Embedding ---
|
||||
const embeddingGroup = obj(c, "embedding");
|
||||
let embeddingConfigError: string | undefined;
|
||||
|
||||
// Embedding config: determine provider based on user input and apiKey availability
|
||||
const embeddingApiKey = str(embeddingGroup, "apiKey") ?? "";
|
||||
const embeddingBaseUrl = str(embeddingGroup, "baseUrl") ?? "";
|
||||
const embeddingProviderRaw = str(embeddingGroup, "provider") ?? "none";
|
||||
const embeddingModelRaw = str(embeddingGroup, "model") ?? "";
|
||||
const embeddingDimensionsRaw = num(embeddingGroup, "dimensions");
|
||||
const embeddingProxyUrl = str(embeddingGroup, "proxyUrl");
|
||||
|
||||
// provider="none" → embedding disabled (default for zero-config users)
|
||||
// provider="local" → no longer exposed to users; treated as disabled at entry level
|
||||
// provider="qclaw" → requires proxyUrl for local proxy forwarding
|
||||
// Any other value → remote mode (requires apiKey, baseUrl, model, dimensions)
|
||||
let embeddingProvider: string;
|
||||
let embeddingEnabled = bool(embeddingGroup, "enabled") ?? true;
|
||||
|
||||
if (embeddingProviderRaw === "none") {
|
||||
// Explicitly disabled (default): no embedding, no vector search
|
||||
embeddingProvider = "none";
|
||||
embeddingEnabled = false;
|
||||
} else if (embeddingProviderRaw === "local") {
|
||||
// Local embedding is not exposed to users; treat as disabled at entry level.
|
||||
// Internal LocalEmbeddingService code is preserved but not reachable from config.
|
||||
embeddingProvider = "none";
|
||||
embeddingEnabled = false;
|
||||
embeddingConfigError =
|
||||
"Local embedding provider is not available in user config. " +
|
||||
"Please configure a remote embedding provider (e.g. openai, deepseek). Embedding has been disabled.";
|
||||
} else if (embeddingProviderRaw === "qclaw") {
|
||||
// qclaw provider: requires proxyUrl for local proxy forwarding
|
||||
const missingFields: string[] = [];
|
||||
if (!embeddingProxyUrl) missingFields.push("proxyUrl");
|
||||
if (!embeddingBaseUrl) missingFields.push("baseUrl");
|
||||
if (!embeddingApiKey) missingFields.push("apiKey");
|
||||
if (!embeddingModelRaw) missingFields.push("model");
|
||||
if (embeddingDimensionsRaw == null || embeddingDimensionsRaw <= 0) missingFields.push("dimensions");
|
||||
|
||||
if (missingFields.length > 0) {
|
||||
const errorMsg =
|
||||
`Embedding provider 'qclaw' requires 'proxyUrl', 'baseUrl', 'apiKey', 'model', and 'dimensions' to be set. ` +
|
||||
`Missing: ${missingFields.join(", ")}. Embedding has been disabled.`;
|
||||
embeddingConfigError = errorMsg;
|
||||
embeddingEnabled = false;
|
||||
embeddingProvider = embeddingProviderRaw;
|
||||
} else {
|
||||
embeddingProvider = embeddingProviderRaw;
|
||||
}
|
||||
} else {
|
||||
// Remote mode — validate all required fields
|
||||
const missingFields: string[] = [];
|
||||
if (!embeddingApiKey) missingFields.push("apiKey");
|
||||
if (!embeddingBaseUrl) missingFields.push("baseUrl");
|
||||
if (!embeddingModelRaw) missingFields.push("model");
|
||||
if (embeddingDimensionsRaw == null || embeddingDimensionsRaw <= 0) missingFields.push("dimensions");
|
||||
|
||||
if (missingFields.length > 0) {
|
||||
// Configuration error: disable embedding and log detailed error
|
||||
// This does NOT throw — the plugin continues running without vector search
|
||||
const errorMsg =
|
||||
`Remote embedding provider '${embeddingProviderRaw}' requires 'apiKey', 'baseUrl', 'model', and 'dimensions' to be set. ` +
|
||||
`Missing: ${missingFields.join(", ")}. Embedding has been disabled.`;
|
||||
// We store the error message so the caller (index.ts) can log it
|
||||
embeddingConfigError = errorMsg;
|
||||
embeddingEnabled = false;
|
||||
embeddingProvider = embeddingProviderRaw; // preserve original for error context
|
||||
} else {
|
||||
embeddingProvider = embeddingProviderRaw;
|
||||
}
|
||||
}
|
||||
|
||||
// When provider="none", dimensions=0 signals VectorStore to skip vec0 table
|
||||
// creation entirely (deferred until a real embedding provider is configured).
|
||||
// This avoids creating vec0 tables with a placeholder dimension that would
|
||||
// mismatch if the user later enables a different-dimensional provider.
|
||||
const defaultDimensions =
|
||||
embeddingProvider === "none" ? 0 :
|
||||
embeddingDimensionsRaw ?? 0;
|
||||
const defaultModel = embeddingProvider === "none" ? "" : embeddingModelRaw;
|
||||
|
||||
const cleanTime = normalizeCleanTime(str(captureGroup, "cleanTime")) ?? "03:00";
|
||||
|
||||
const memoryCleanup: MemoryCleanupConfig = {
|
||||
retentionDays,
|
||||
enabled: retentionDays != null,
|
||||
cleanTime,
|
||||
};
|
||||
|
||||
return {
|
||||
capture: {
|
||||
enabled: bool(captureGroup, "enabled") ?? true,
|
||||
excludeAgents: strArray(captureGroup, "excludeAgents") ?? [],
|
||||
l0l1RetentionDays: retentionDays ?? 0,
|
||||
allowAggressiveCleanup,
|
||||
},
|
||||
extraction: {
|
||||
enabled: bool(extractionGroup, "enabled") ?? true,
|
||||
enableDedup: bool(extractionGroup, "enableDedup") ?? true,
|
||||
maxMemoriesPerSession: num(extractionGroup, "maxMemoriesPerSession") ?? 20,
|
||||
model: optStr(extractionGroup, "model"),
|
||||
},
|
||||
persona: {
|
||||
triggerEveryN: num(personaGroup, "triggerEveryN") ?? 50,
|
||||
maxScenes: num(personaGroup, "maxScenes") ?? 20,
|
||||
backupCount: num(personaGroup, "backupCount") ?? 3,
|
||||
sceneBackupCount: num(personaGroup, "sceneBackupCount") ?? 10,
|
||||
model: optStr(personaGroup, "model"),
|
||||
},
|
||||
pipeline: {
|
||||
everyNConversations: num(pipelineGroup, "everyNConversations") ?? 5,
|
||||
enableWarmup: bool(pipelineGroup, "enableWarmup") ?? true,
|
||||
l1IdleTimeoutSeconds: num(pipelineGroup, "l1IdleTimeoutSeconds") ?? 60,
|
||||
l2DelayAfterL1Seconds: num(pipelineGroup, "l2DelayAfterL1Seconds") ?? 90,
|
||||
l2MinIntervalSeconds: num(pipelineGroup, "l2MinIntervalSeconds") ?? 300,
|
||||
l2MaxIntervalSeconds: num(pipelineGroup, "l2MaxIntervalSeconds") ?? 1800,
|
||||
sessionActiveWindowHours: num(pipelineGroup, "sessionActiveWindowHours") ?? 24,
|
||||
},
|
||||
recall: {
|
||||
enabled: bool(recallGroup, "enabled") ?? true,
|
||||
maxResults: num(recallGroup, "maxResults") ?? 5,
|
||||
scoreThreshold: num(recallGroup, "scoreThreshold") ?? 0.3,
|
||||
strategy: validateStrategy(str(recallGroup, "strategy")) ?? "hybrid",
|
||||
timeoutMs: num(recallGroup, "timeoutMs") ?? 5000,
|
||||
},
|
||||
embedding: {
|
||||
enabled: embeddingEnabled,
|
||||
provider: embeddingProvider,
|
||||
baseUrl: embeddingBaseUrl,
|
||||
apiKey: embeddingApiKey,
|
||||
model: str(embeddingGroup, "model") ?? defaultModel,
|
||||
dimensions: num(embeddingGroup, "dimensions") ?? defaultDimensions,
|
||||
conflictRecallTopK: num(embeddingGroup, "conflictRecallTopK") ?? 5,
|
||||
proxyUrl: embeddingProxyUrl,
|
||||
maxInputChars: num(embeddingGroup, "maxInputChars") ?? 5000,
|
||||
timeoutMs: num(embeddingGroup, "timeoutMs") ?? 10_000,
|
||||
modelCacheDir: optStr(embeddingGroup, "modelCacheDir"),
|
||||
configError: embeddingConfigError,
|
||||
},
|
||||
memoryCleanup,
|
||||
report: {
|
||||
enabled: bool(obj(c, "report"), "enabled") ?? false,
|
||||
type: str(obj(c, "report"), "type") ?? "local",
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Helper functions
|
||||
// ============================
|
||||
|
||||
/** Get sub-object by key, or empty object if missing. */
|
||||
function obj(c: Record<string, unknown>, key: string): Record<string, unknown> {
|
||||
const v = c[key];
|
||||
return v && typeof v === "object" && !Array.isArray(v) ? v as Record<string, unknown> : {};
|
||||
}
|
||||
|
||||
function str(src: Record<string, unknown>, key: string): string | undefined {
|
||||
const v = src[key];
|
||||
return typeof v === "string" && v.trim() ? v.trim() : undefined;
|
||||
}
|
||||
|
||||
function optStr(src: Record<string, unknown>, key: string): string | undefined {
|
||||
const v = src[key];
|
||||
return typeof v === "string" ? v : undefined;
|
||||
}
|
||||
|
||||
function num(src: Record<string, unknown>, key: string): number | undefined {
|
||||
const v = src[key];
|
||||
return typeof v === "number" && Number.isFinite(v) ? v : undefined;
|
||||
}
|
||||
|
||||
function bool(src: Record<string, unknown>, key: string): boolean | undefined {
|
||||
const v = src[key];
|
||||
return typeof v === "boolean" ? v : undefined;
|
||||
}
|
||||
|
||||
function strArray(src: Record<string, unknown>, key: string): string[] | undefined {
|
||||
const v = src[key];
|
||||
if (!Array.isArray(v)) return undefined;
|
||||
return v.filter((item): item is string => typeof item === "string" && item.trim().length > 0);
|
||||
}
|
||||
|
||||
const VALID_STRATEGIES: RecallConfig["strategy"][] = ["embedding", "keyword", "hybrid"];
|
||||
|
||||
/**
|
||||
* Validate recall strategy against whitelist.
|
||||
* Returns the strategy if valid, undefined otherwise (caller falls back to default).
|
||||
*/
|
||||
function validateStrategy(value: string | undefined): RecallConfig["strategy"] | undefined {
|
||||
if (!value) return undefined;
|
||||
return VALID_STRATEGIES.includes(value as RecallConfig["strategy"])
|
||||
? (value as RecallConfig["strategy"])
|
||||
: undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize a cleanup time string.
|
||||
*
|
||||
* The input must follow "HH:MM" or "H:MM" format (24-hour clock).
|
||||
* If the time is valid, it returns the normalized format "HH:MM"
|
||||
* with leading zeros added when necessary.
|
||||
* If the format is invalid or the time is out of range
|
||||
* (hour: 0–23, minute: 0–59), it returns undefined.
|
||||
*
|
||||
* Examples:
|
||||
* normalizeCleanTime("3:05") -> "03:05"
|
||||
* normalizeCleanTime("03:05") -> "03:05"
|
||||
* normalizeCleanTime("23:59") -> "23:59"
|
||||
*
|
||||
* normalizeCleanTime("24:00") -> undefined // hour out of range
|
||||
* normalizeCleanTime("12:60") -> undefined // minute out of range
|
||||
* normalizeCleanTime("3:5") -> undefined // minute must have two digits
|
||||
* normalizeCleanTime("abc") -> undefined // invalid format
|
||||
*/
|
||||
function normalizeCleanTime(input: string | undefined): string | undefined {
|
||||
if (!input) return undefined;
|
||||
const trimmed = input.trim();
|
||||
const m = /^(\d{1,2}):(\d{2})$/.exec(trimmed);
|
||||
if (!m) return undefined;
|
||||
|
||||
const hh = Number(m[1]);
|
||||
const mm = Number(m[2]);
|
||||
if (!Number.isInteger(hh) || !Number.isInteger(mm)) return undefined;
|
||||
if (hh < 0 || hh > 23 || mm < 0 || mm > 59) return undefined;
|
||||
|
||||
return `${String(hh).padStart(2, "0")}:${String(mm).padStart(2, "0")}`;
|
||||
}
|
||||
@@ -0,0 +1,579 @@
|
||||
/**
|
||||
* L0 Conversation Recorder: records raw conversation messages to local JSONL files.
|
||||
*
|
||||
* Triggered from agent_end hook. Receives the conversation messages directly from
|
||||
* the hook context (no file I/O needed), sanitizes them, filters out noise, and
|
||||
* writes to ~/.openclaw/memory-tdai/conversations/YYYY-MM-DD.jsonl
|
||||
*
|
||||
* Design decisions:
|
||||
* - Uses JSONL format (**one message per line** — flat, easy to grep/stream)
|
||||
* - One file per day (all sessions merged into the same daily file)
|
||||
* - sessionKey is stored as a field in each JSONL line, not in the filename
|
||||
* - Independent from system session files — format fully controlled by plugin
|
||||
* - Messages are sanitized to remove injected tags (prevent feedback loops)
|
||||
* - Short/long/command messages are filtered out
|
||||
*/
|
||||
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import crypto from "node:crypto";
|
||||
import { sanitizeText, stripCodeBlocks, shouldCaptureL0 } from "../utils/sanitize.js";
|
||||
|
||||
// ============================
|
||||
// Types
|
||||
// ============================
|
||||
|
||||
export interface ConversationMessage {
|
||||
/** Unique message ID (used by L1 prompt for source_message_ids tracking) */
|
||||
id: string;
|
||||
role: "user" | "assistant";
|
||||
content: string;
|
||||
timestamp: number; // epoch ms
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a short unique message ID.
|
||||
*/
|
||||
function generateMessageId(): string {
|
||||
return `msg_${Date.now()}_${crypto.randomBytes(3).toString("hex")}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* New flat format: one message per JSONL line.
|
||||
*/
|
||||
export interface L0MessageRecord {
|
||||
sessionKey: string;
|
||||
sessionId: string;
|
||||
recordedAt: string; // ISO timestamp
|
||||
id: string;
|
||||
role: "user" | "assistant";
|
||||
content: string;
|
||||
timestamp: number; // epoch ms
|
||||
}
|
||||
|
||||
/**
|
||||
* A group of conversation messages (used by downstream consumers).
|
||||
* Each L0ConversationRecord represents one or more messages from the same recording event.
|
||||
*/
|
||||
export interface L0ConversationRecord {
|
||||
sessionKey: string;
|
||||
sessionId: string;
|
||||
recordedAt: string; // ISO timestamp
|
||||
messageCount: number;
|
||||
messages: ConversationMessage[];
|
||||
}
|
||||
|
||||
interface Logger {
|
||||
debug?: (message: string) => void;
|
||||
info: (message: string) => void;
|
||||
warn: (message: string) => void;
|
||||
error: (message: string) => void;
|
||||
}
|
||||
|
||||
const TAG = "[memory-tdai][l0]";
|
||||
|
||||
// ============================
|
||||
// Core function
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Record a conversation round to the L0 JSONL file.
|
||||
*
|
||||
* Only records **incremental** messages (new since the last capture).
|
||||
* Uses `afterTimestamp` as the primary filter to skip already-captured history.
|
||||
*
|
||||
* @param sessionKey - The session key for this conversation
|
||||
* @param rawMessages - Raw messages from the agent_end hook context (full session history)
|
||||
* @param baseDir - Base data directory (~/.openclaw/memory-tdai/)
|
||||
* @param logger - Optional logger
|
||||
* @param originalUserText - Clean original user prompt (pre-prependContext)
|
||||
* @param afterTimestamp - Epoch ms cursor: only messages with timestamp > this are new.
|
||||
* Pass 0 or omit for the first capture of a session.
|
||||
* @returns Filtered messages (for L1 to use directly), or empty array if nothing worth recording
|
||||
*/
|
||||
export async function recordConversation(params: {
|
||||
sessionKey: string;
|
||||
sessionId?: string;
|
||||
rawMessages: unknown[];
|
||||
baseDir: string;
|
||||
logger?: Logger;
|
||||
/** Clean original user prompt (pre-prependContext) */
|
||||
originalUserText?: string;
|
||||
/** Epoch ms cursor: only process messages with timestamp strictly greater than this. */
|
||||
afterTimestamp?: number;
|
||||
/**
|
||||
* Number of messages in the session at before_prompt_build time.
|
||||
* Used to locate the exact user message that originalUserText corresponds to:
|
||||
* rawMessages[originalUserMessageCount] is the user message appended by the framework
|
||||
* AFTER before_prompt_build, i.e. the one whose content was polluted by prependContext.
|
||||
*/
|
||||
originalUserMessageCount?: number;
|
||||
}): Promise<ConversationMessage[]> {
|
||||
const { sessionKey, sessionId, rawMessages, baseDir, logger, originalUserText, afterTimestamp, originalUserMessageCount } = params;
|
||||
|
||||
// Step 1: Position slice + extract user/assistant messages.
|
||||
//
|
||||
// Dual protection against duplicate capture:
|
||||
// Layer 1 (position slice): Use originalUserMessageCount (cached at before_prompt_build)
|
||||
// to slice rawMessages — only keep messages added AFTER the prompt build, i.e. this
|
||||
// turn's new messages. This is immune to timestamp drift after gateway restarts.
|
||||
// Layer 2 (timestamp cursor): The existing afterTimestamp filter below acts as a fallback
|
||||
// when the position slice is unavailable (cache expired, process restart, etc.).
|
||||
const usePositionSlice = originalUserMessageCount != null && originalUserMessageCount > 0
|
||||
&& originalUserMessageCount <= rawMessages.length;
|
||||
const slicedMessages = usePositionSlice
|
||||
? rawMessages.slice(originalUserMessageCount)
|
||||
: rawMessages;
|
||||
|
||||
const allExtracted = extractUserAssistantMessages(slicedMessages);
|
||||
|
||||
if (usePositionSlice) {
|
||||
logger?.debug?.(
|
||||
`${TAG} Position slice: ${rawMessages.length} raw → ${slicedMessages.length} new (sliceStart=${originalUserMessageCount})`,
|
||||
);
|
||||
}
|
||||
|
||||
// Diagnostic: check whether the framework actually provides timestamp on raw messages.
|
||||
// If all raw timestamps are missing, the timestamp cursor is effectively useless and
|
||||
// position slice becomes the sole incremental mechanism.
|
||||
if (slicedMessages.length > 0) {
|
||||
const firstRaw = slicedMessages[0] as Record<string, unknown> | undefined;
|
||||
const rawTs = firstRaw?.timestamp;
|
||||
const hasRawTs = typeof rawTs === "number";
|
||||
logger?.debug?.(
|
||||
`${TAG} Raw message[0] timestamp probe: ${hasRawTs ? `present (${rawTs})` : `missing (type=${typeof rawTs}, value=${String(rawTs)})`}`,
|
||||
);
|
||||
}
|
||||
|
||||
logger?.debug?.(`${TAG} Extracted ${allExtracted.length} user/assistant messages from ${slicedMessages.length} total`);
|
||||
|
||||
// Step 1.5: Incremental filter — only keep messages newer than the cursor.
|
||||
//
|
||||
// Uses strict greater-than (>) which is safe because:
|
||||
// - The cursor is set to max(timestamps) of the LAST recorded batch.
|
||||
// - The next agent turn's messages will have timestamps strictly greater than
|
||||
// the previous turn (there's at least one LLM API call between turns, which
|
||||
// takes hundreds of milliseconds minimum — no same-millisecond collision).
|
||||
// - All messages within a single turn are captured together as one batch,
|
||||
// so even if multiple messages share the same timestamp, they are either
|
||||
// all included (new batch) or all excluded (already captured).
|
||||
// - If a message lacks a timestamp field, extractUserAssistantMessages()
|
||||
// assigns Date.now() at extraction time, which is always > previous cursor.
|
||||
const cursor = afterTimestamp ?? 0;
|
||||
const extracted = cursor > 0
|
||||
? allExtracted.filter((m) => m.timestamp > cursor)
|
||||
: allExtracted;
|
||||
|
||||
if (extracted.length > 0) {
|
||||
const first = extracted[0];
|
||||
logger?.debug?.(
|
||||
`${TAG} First captured message: role=${first.role}, ts=${first.timestamp}, ` +
|
||||
`date=${new Date(first.timestamp).toISOString()}, content=${first.content.slice(0, 80)}${first.content.length > 80 ? "…" : ""}`,
|
||||
);
|
||||
}
|
||||
|
||||
if (cursor > 0) {
|
||||
logger?.debug?.(
|
||||
`${TAG} Incremental filter: ${allExtracted.length} total → ${extracted.length} new (cursor=${cursor})`,
|
||||
);
|
||||
|
||||
// Safety valve: if timestamp filter passed everything through and position slice
|
||||
// was not available, this likely indicates timestamp drift after a gateway restart.
|
||||
if (!usePositionSlice && extracted.length === allExtracted.length && allExtracted.length > 8) {
|
||||
logger?.warn?.(
|
||||
`${TAG} ⚠ Safety valve: all ${allExtracted.length} messages passed timestamp filter (cursor=${cursor}) — ` +
|
||||
`possible timestamp drift after gateway restart. Position slice was not available (no cached messageCount).`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (extracted.length === 0) {
|
||||
logger?.debug?.(`${TAG} No new user/assistant messages to record`);
|
||||
return [];
|
||||
}
|
||||
|
||||
// Step 2: Replace polluted user messages with cached original prompt.
|
||||
//
|
||||
// Background:
|
||||
// The framework appends the user's message to the session after before_prompt_build,
|
||||
// then injects prependContext into it. So the user message in rawMessages is polluted.
|
||||
// We cached the clean prompt (originalUserText) and the message count at
|
||||
// before_prompt_build time (originalUserMessageCount) to identify which raw message
|
||||
// is the real user input.
|
||||
//
|
||||
// Strategy:
|
||||
// When position slice is active, the polluted user message is slicedMessages[0].
|
||||
// Otherwise, fall back to rawMessages[originalUserMessageCount].
|
||||
// In both cases, find the timestamp and match it in `extracted` for replacement.
|
||||
// If matching fails, skip replacement — sanitizeText() in Step 3 is the safety net.
|
||||
if (originalUserText) {
|
||||
// Determine the target raw message that contains the polluted user prompt
|
||||
const targetRaw: Record<string, unknown> | undefined = usePositionSlice
|
||||
? slicedMessages[0] as Record<string, unknown> | undefined
|
||||
: (originalUserMessageCount != null && originalUserMessageCount >= 0 && originalUserMessageCount < rawMessages.length)
|
||||
? rawMessages[originalUserMessageCount] as Record<string, unknown> | undefined
|
||||
: undefined;
|
||||
|
||||
const targetTs = targetRaw && typeof targetRaw.timestamp === "number" ? targetRaw.timestamp : undefined;
|
||||
|
||||
if (targetTs != null) {
|
||||
let replaced = false;
|
||||
for (let i = 0; i < extracted.length; i++) {
|
||||
if (extracted[i].role === "user" && extracted[i].timestamp === targetTs) {
|
||||
logger?.debug?.(
|
||||
`${TAG} Replacing user message at timestamp=${targetTs} with cached original prompt ` +
|
||||
`(${originalUserText.length} chars, was ${extracted[i].content.length} chars) [positionSlice=${usePositionSlice}]`,
|
||||
);
|
||||
extracted[i] = { ...extracted[i], content: originalUserText };
|
||||
replaced = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!replaced) {
|
||||
logger?.warn?.(
|
||||
`${TAG} Target user message (ts=${targetTs}) not found in extracted batch — ` +
|
||||
`possibly filtered by cursor. Skipping replacement, will rely on sanitizeText().`,
|
||||
);
|
||||
}
|
||||
} else if (targetRaw) {
|
||||
logger?.warn?.(
|
||||
`${TAG} Target raw message has no valid timestamp — ` +
|
||||
`skipping replacement, will rely on sanitizeText().`,
|
||||
);
|
||||
} else {
|
||||
logger?.warn?.(
|
||||
`${TAG} Have originalUserText but cannot locate target raw message — ` +
|
||||
`skipping replacement, will rely on sanitizeText().`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Step 3: Sanitize and filter
|
||||
const filtered = extracted
|
||||
.map((m) => {
|
||||
let content = sanitizeText(m.content);
|
||||
// Strip fenced code blocks from assistant replies to reduce embedding noise
|
||||
if (m.role === "assistant") {
|
||||
content = stripCodeBlocks(content);
|
||||
}
|
||||
return { id: m.id, role: m.role, content, timestamp: m.timestamp };
|
||||
})
|
||||
.filter((m) => shouldCaptureL0(m.content));
|
||||
|
||||
logger?.debug?.(`${TAG} After sanitize+filter: ${filtered.length} messages (from ${extracted.length})`);
|
||||
|
||||
if (filtered.length === 0) {
|
||||
logger?.debug?.(`${TAG} All messages filtered out, skipping L0 write`);
|
||||
return [];
|
||||
}
|
||||
|
||||
// Step 4: Write to JSONL file — one message per line (flat format)
|
||||
const now = new Date().toISOString();
|
||||
const lines: string[] = [];
|
||||
for (const msg of filtered) {
|
||||
const record: L0MessageRecord = {
|
||||
sessionKey,
|
||||
sessionId: sessionId || "",
|
||||
recordedAt: now,
|
||||
id: msg.id,
|
||||
role: msg.role,
|
||||
content: msg.content,
|
||||
timestamp: msg.timestamp,
|
||||
};
|
||||
lines.push(JSON.stringify(record));
|
||||
}
|
||||
|
||||
const shardDate = formatLocalDate(new Date());
|
||||
const outDir = path.join(baseDir, "conversations");
|
||||
const outPath = path.join(outDir, `${shardDate}.jsonl`);
|
||||
|
||||
try {
|
||||
await fs.mkdir(outDir, { recursive: true });
|
||||
// Append each message as its own JSONL line
|
||||
await fs.appendFile(outPath, lines.join("\n") + "\n", "utf-8");
|
||||
logger?.debug?.(`${TAG} Recorded ${filtered.length} messages to ${outPath}`);
|
||||
} catch (err) {
|
||||
logger?.error(`${TAG} Failed to write L0 file: ${err instanceof Error ? err.message : String(err)}`);
|
||||
// Return filtered messages anyway so L1 can still process them
|
||||
}
|
||||
|
||||
return filtered;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read all L0 conversation records for a session.
|
||||
* Returns records in chronological order.
|
||||
*
|
||||
* File format: `YYYY-MM-DD.jsonl` (daily files, all sessions merged).
|
||||
* Each line is an L0MessageRecord; filtered by sessionKey at line level.
|
||||
*/
|
||||
export async function readConversationRecords(
|
||||
sessionKey: string,
|
||||
baseDir: string,
|
||||
logger?: Logger,
|
||||
): Promise<L0ConversationRecord[]> {
|
||||
const conversationsDir = path.join(baseDir, "conversations");
|
||||
|
||||
// Daily file pattern: YYYY-MM-DD.jsonl
|
||||
const dateFilePattern = /^\d{4}-\d{2}-\d{2}\.jsonl$/;
|
||||
|
||||
let entries: string[];
|
||||
try {
|
||||
const dirEntries = await fs.readdir(conversationsDir, { withFileTypes: true });
|
||||
entries = dirEntries
|
||||
.filter((entry) => entry.isFile())
|
||||
.map((entry) => entry.name);
|
||||
} catch {
|
||||
// Directory doesn't exist yet — normal for first conversation
|
||||
return [];
|
||||
}
|
||||
|
||||
const targetFiles = entries
|
||||
.filter((name) => dateFilePattern.test(name))
|
||||
.sort();
|
||||
|
||||
if (targetFiles.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const records: L0ConversationRecord[] = [];
|
||||
|
||||
for (const fileName of targetFiles) {
|
||||
const filePath = path.join(conversationsDir, fileName);
|
||||
|
||||
let raw: string;
|
||||
try {
|
||||
raw = await fs.readFile(filePath, "utf-8");
|
||||
} catch {
|
||||
logger?.warn?.(`${TAG} Failed to read L0 file: ${filePath}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const lines = raw.split("\n").filter((line: string) => line.trim());
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const line = lines[i];
|
||||
try {
|
||||
const parsed = JSON.parse(line) as Record<string, unknown>;
|
||||
|
||||
// Filter by sessionKey at line level
|
||||
const lineSessionKey = parsed.sessionKey as string | undefined;
|
||||
if (lineSessionKey !== sessionKey) continue;
|
||||
|
||||
if (typeof parsed.role === "string" && typeof parsed.content === "string") {
|
||||
// Flat format: { sessionKey, sessionId, recordedAt, id, role, content, timestamp }
|
||||
// Wrap into L0ConversationRecord for uniform downstream consumption
|
||||
const msg: ConversationMessage = {
|
||||
id: (typeof parsed.id === "string" && parsed.id) ? parsed.id : generateMessageId(),
|
||||
role: parsed.role as "user" | "assistant",
|
||||
content: parsed.content as string,
|
||||
timestamp: typeof parsed.timestamp === "number" ? parsed.timestamp : Date.now(),
|
||||
};
|
||||
records.push({
|
||||
sessionKey: (parsed.sessionKey as string) || sessionKey,
|
||||
sessionId: (parsed.sessionId as string) || "",
|
||||
recordedAt: (parsed.recordedAt as string) || new Date().toISOString(),
|
||||
messageCount: 1,
|
||||
messages: [msg],
|
||||
});
|
||||
} else {
|
||||
logger?.warn?.(`${TAG} Unrecognized JSONL line format in ${filePath}:${i + 1}`);
|
||||
}
|
||||
} catch {
|
||||
logger?.warn?.(`${TAG} Skipping malformed JSONL line in ${filePath}:${i + 1}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
records.sort((a, b) => {
|
||||
const ta = Date.parse(a.recordedAt);
|
||||
const tb = Date.parse(b.recordedAt);
|
||||
const na = Number.isFinite(ta) ? ta : Number.POSITIVE_INFINITY;
|
||||
const nb = Number.isFinite(tb) ? tb : Number.POSITIVE_INFINITY;
|
||||
return na - nb;
|
||||
});
|
||||
|
||||
return records;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read L0 messages across all conversation records for a session,
|
||||
* optionally filtered by a cursor timestamp (messages after the cursor).
|
||||
*
|
||||
* When `limit` is provided, only the **newest** `limit` messages are returned
|
||||
* (matching the DB path's `ORDER BY timestamp DESC LIMIT ?` behavior).
|
||||
* Returned messages are always in chronological order (oldest → newest).
|
||||
*
|
||||
* NOTE: potential optimization — records are chronologically ordered (append-only JSONL),
|
||||
* so a reverse scan could skip entire old records. Deferred for now; see Issue 5 in
|
||||
* docs/05-known-issues.md.
|
||||
*/
|
||||
export async function readConversationMessages(
|
||||
sessionKey: string,
|
||||
baseDir: string,
|
||||
afterTimestamp?: number,
|
||||
logger?: Logger,
|
||||
limit?: number,
|
||||
): Promise<ConversationMessage[]> {
|
||||
const records = await readConversationRecords(sessionKey, baseDir, logger);
|
||||
const allMessages: ConversationMessage[] = [];
|
||||
|
||||
for (const record of records) {
|
||||
for (const msg of record.messages) {
|
||||
if (afterTimestamp && msg.timestamp <= afterTimestamp) continue;
|
||||
allMessages.push(msg);
|
||||
}
|
||||
}
|
||||
|
||||
// Truncate to newest `limit` messages (keep tail, since array is chronological)
|
||||
if (limit != null && limit > 0 && allMessages.length > limit) {
|
||||
logger?.debug?.(
|
||||
`${TAG} readConversationMessages: truncating ${allMessages.length} → ${limit} (newest)`,
|
||||
);
|
||||
return allMessages.slice(-limit);
|
||||
}
|
||||
|
||||
return allMessages;
|
||||
}
|
||||
|
||||
/**
|
||||
* A group of conversation messages sharing the same sessionId.
|
||||
*/
|
||||
export interface SessionIdMessageGroup {
|
||||
sessionId: string;
|
||||
messages: ConversationMessage[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Read L0 messages for a session, grouped by sessionId.
|
||||
*
|
||||
* Within the same sessionKey, different sessionIds represent different conversation
|
||||
* instances (e.g. after /reset). L1 extraction should process each group independently
|
||||
* so that each group's sessionId is correctly associated with its extracted memories.
|
||||
*
|
||||
* When `limit` is provided, only the **newest** `limit` messages (across all groups)
|
||||
* are retained — matching the DB path's `ORDER BY timestamp DESC LIMIT ?` behavior.
|
||||
* Groups that become empty after truncation are dropped.
|
||||
*
|
||||
* Groups are returned in chronological order (by earliest message timestamp).
|
||||
* Messages within each group are also in chronological order.
|
||||
*/
|
||||
export async function readConversationMessagesGroupedBySessionId(
|
||||
sessionKey: string,
|
||||
baseDir: string,
|
||||
afterTimestamp?: number,
|
||||
logger?: Logger,
|
||||
limit?: number,
|
||||
): Promise<SessionIdMessageGroup[]> {
|
||||
const records = await readConversationRecords(sessionKey, baseDir, logger);
|
||||
|
||||
// Collect all messages with their sessionId, respecting afterTimestamp filter
|
||||
const allMessages: Array<{ sessionId: string; msg: ConversationMessage }> = [];
|
||||
|
||||
for (const record of records) {
|
||||
const sid = record.sessionId || "";
|
||||
for (const msg of record.messages) {
|
||||
if (afterTimestamp && msg.timestamp <= afterTimestamp) continue;
|
||||
allMessages.push({ sessionId: sid, msg });
|
||||
}
|
||||
}
|
||||
|
||||
// Sort by timestamp ASC (chronological) — records are already roughly ordered
|
||||
// by recordedAt, but messages within may not be perfectly sorted by timestamp.
|
||||
allMessages.sort((a, b) => a.msg.timestamp - b.msg.timestamp);
|
||||
|
||||
// Truncate to newest `limit` messages (keep tail)
|
||||
let selected = allMessages;
|
||||
if (limit != null && limit > 0 && allMessages.length > limit) {
|
||||
logger?.debug?.(
|
||||
`${TAG} readConversationMessagesGroupedBySessionId: truncating ${allMessages.length} → ${limit} (newest)`,
|
||||
);
|
||||
selected = allMessages.slice(-limit);
|
||||
}
|
||||
|
||||
// Re-group by sessionId
|
||||
const groupMap = new Map<string, ConversationMessage[]>();
|
||||
for (const { sessionId, msg } of selected) {
|
||||
let group = groupMap.get(sessionId);
|
||||
if (!group) {
|
||||
group = [];
|
||||
groupMap.set(sessionId, group);
|
||||
}
|
||||
group.push(msg);
|
||||
}
|
||||
|
||||
// Convert to array, sorted by earliest message timestamp in each group
|
||||
const groups: SessionIdMessageGroup[] = [];
|
||||
for (const [sessionId, messages] of groupMap) {
|
||||
if (messages.length > 0) {
|
||||
groups.push({ sessionId, messages });
|
||||
}
|
||||
}
|
||||
groups.sort((a, b) => a.messages[0].timestamp - b.messages[0].timestamp);
|
||||
|
||||
return groups;
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Helpers
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Extract user and assistant messages from raw hook message array.
|
||||
*/
|
||||
function extractUserAssistantMessages(messages: unknown[]): ConversationMessage[] {
|
||||
const result: ConversationMessage[] = [];
|
||||
|
||||
for (const msg of messages) {
|
||||
if (!msg || typeof msg !== "object") continue;
|
||||
const m = msg as Record<string, unknown>;
|
||||
const role = m.role as string | undefined;
|
||||
|
||||
if (role !== "user" && role !== "assistant") continue;
|
||||
|
||||
let content: string | undefined;
|
||||
if (typeof m.content === "string") {
|
||||
content = m.content;
|
||||
} else if (Array.isArray(m.content)) {
|
||||
const textParts: string[] = [];
|
||||
for (const part of m.content) {
|
||||
if (
|
||||
part &&
|
||||
typeof part === "object" &&
|
||||
(part as Record<string, unknown>).type === "text"
|
||||
) {
|
||||
const text = (part as Record<string, unknown>).text;
|
||||
if (typeof text === "string") textParts.push(text);
|
||||
}
|
||||
}
|
||||
content = textParts.join("\n");
|
||||
}
|
||||
|
||||
// Strip inline base64 image data URIs that some providers embed in string content.
|
||||
// These are not useful for memory and would pollute FTS / embedding indexes.
|
||||
if (content && /data:image\/[a-z+]+;base64,/i.test(content)) {
|
||||
content = content.replace(/data:image\/[a-z+]+;base64,[A-Za-z0-9+/=]+/gi, "[image]");
|
||||
}
|
||||
|
||||
if (content && content.trim()) {
|
||||
const ts = typeof m.timestamp === "number" ? m.timestamp : Date.now();
|
||||
result.push({
|
||||
id: (typeof m.id === "string" && m.id) ? m.id : generateMessageId(),
|
||||
role: role as "user" | "assistant",
|
||||
content: content.trim(),
|
||||
timestamp: ts,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Format local date as YYYY-MM-DD.
|
||||
*/
|
||||
function formatLocalDate(d: Date): string {
|
||||
const y = d.getFullYear();
|
||||
const m = String(d.getMonth() + 1).padStart(2, "0");
|
||||
const day = String(d.getDate()).padStart(2, "0");
|
||||
return `${y}-${m}-${day}`;
|
||||
}
|
||||
@@ -0,0 +1,270 @@
|
||||
/**
|
||||
* auto-capture hook (v3): records conversation messages locally (L0),
|
||||
* then notifies the MemoryPipelineManager for L1/L2/L3 scheduling.
|
||||
*
|
||||
* Key design decisions:
|
||||
* - Always write L0 locally via l0-recorder.
|
||||
* - When VectorStore + EmbeddingService are available, also write L0 vector index.
|
||||
* - Notify MemoryPipelineManager for L1/L2/L3 trigger evaluation.
|
||||
* - L1 Runner reads from VectorStore DB (primary) or L0 JSONL files (fallback).
|
||||
* - Extraction is NOT triggered here. The pipeline manager decides when.
|
||||
*/
|
||||
|
||||
import crypto from "node:crypto";
|
||||
import type { MemoryTdaiConfig } from "../config.js";
|
||||
import { CheckpointManager } from "../utils/checkpoint.js";
|
||||
import type { MemoryPipelineManager } from "../utils/pipeline-manager.js";
|
||||
import { recordConversation } from "../conversation/l0-recorder.js";
|
||||
import type { ConversationMessage } from "../conversation/l0-recorder.js";
|
||||
import type { VectorStore, L0VectorRecord } from "../store/vector-store.js";
|
||||
import type { EmbeddingService } from "../store/embedding.js";
|
||||
|
||||
const TAG = "[memory-tdai] [capture]";
|
||||
|
||||
interface Logger {
|
||||
debug?: (message: string) => void;
|
||||
info: (message: string) => void;
|
||||
warn: (message: string) => void;
|
||||
error: (message: string) => void;
|
||||
}
|
||||
|
||||
export interface AutoCaptureResult {
|
||||
/** Whether the scheduler was notified (conversation count incremented) */
|
||||
schedulerNotified: boolean;
|
||||
/** Number of messages recorded to L0 */
|
||||
l0RecordedCount: number;
|
||||
/** Number of L0 message vectors written */
|
||||
l0VectorsWritten: number;
|
||||
/** Filtered messages for L1 immediate use */
|
||||
filteredMessages: ConversationMessage[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a unique L0 record ID for vector indexing.
|
||||
* Includes an index to distinguish multiple messages within the same round.
|
||||
*/
|
||||
function generateL0RecordId(sessionKey: string, index: number): string {
|
||||
return `l0_${sessionKey}_${Date.now()}_${index}_${crypto.randomBytes(3).toString("hex")}`;
|
||||
}
|
||||
|
||||
export async function performAutoCapture(params: {
|
||||
messages: unknown[];
|
||||
sessionKey: string;
|
||||
sessionId?: string;
|
||||
cfg: MemoryTdaiConfig;
|
||||
pluginDataDir: string;
|
||||
logger?: Logger;
|
||||
scheduler?: MemoryPipelineManager;
|
||||
/** Clean original user prompt from before_prompt_build cache (pre-prependContext). */
|
||||
originalUserText?: string;
|
||||
/**
|
||||
* Number of messages in the session at before_prompt_build time.
|
||||
* Used by l0-recorder to locate the exact user message that originalUserText
|
||||
* corresponds to: rawMessages[originalUserMessageCount] is the polluted user message.
|
||||
*/
|
||||
originalUserMessageCount?: number;
|
||||
/** Epoch ms when the plugin was registered (cold-start time).
|
||||
* Used as fallback cursor when checkpoint has no prior timestamp —
|
||||
* prevents the first agent_end from dumping all session history into L0. */
|
||||
pluginStartTimestamp?: number;
|
||||
/** VectorStore for L0 vector indexing (optional). */
|
||||
vectorStore?: VectorStore;
|
||||
/** EmbeddingService for L0 vector indexing (optional). */
|
||||
embeddingService?: EmbeddingService;
|
||||
}): Promise<AutoCaptureResult> {
|
||||
const {
|
||||
messages, sessionKey, sessionId, cfg, pluginDataDir, logger, scheduler,
|
||||
originalUserText, originalUserMessageCount, pluginStartTimestamp,
|
||||
vectorStore, embeddingService,
|
||||
} = params;
|
||||
const tCaptureStart = performance.now();
|
||||
|
||||
const checkpoint = new CheckpointManager(pluginDataDir, logger);
|
||||
|
||||
// ============================
|
||||
// Step 1 + 2: L0 recording + checkpoint update (ATOMIC)
|
||||
// ============================
|
||||
// These steps are combined inside captureAtomically() to prevent the race
|
||||
// condition where two concurrent agent_end events both read the same stale
|
||||
// cursor and produce duplicate L0 records. The file lock is held for the
|
||||
// entire read-cursor → recordConversation → advance-cursor sequence.
|
||||
const tL0RecordStart = performance.now();
|
||||
let filteredMessages: ConversationMessage[] = [];
|
||||
try {
|
||||
await checkpoint.captureAtomically(
|
||||
sessionKey,
|
||||
pluginStartTimestamp,
|
||||
async (afterTimestamp) => {
|
||||
logger?.debug?.(`${TAG} L0 capture cursor (per-session, atomic): afterTimestamp=${afterTimestamp} session=${sessionKey}`);
|
||||
|
||||
if (afterTimestamp === pluginStartTimestamp && pluginStartTimestamp && pluginStartTimestamp > 0) {
|
||||
logger?.debug?.(
|
||||
`${TAG} No per-session checkpoint cursor found for session=${sessionKey} — ` +
|
||||
`using pluginStartTimestamp as floor: ` +
|
||||
`${afterTimestamp} (${new Date(afterTimestamp).toISOString()})`,
|
||||
);
|
||||
}
|
||||
|
||||
filteredMessages = await recordConversation({
|
||||
sessionKey,
|
||||
sessionId,
|
||||
rawMessages: messages,
|
||||
baseDir: pluginDataDir,
|
||||
logger,
|
||||
originalUserText,
|
||||
afterTimestamp,
|
||||
originalUserMessageCount,
|
||||
});
|
||||
|
||||
if (filteredMessages.length === 0) {
|
||||
return null; // Nothing captured — cursor stays unchanged
|
||||
}
|
||||
|
||||
logger?.debug?.(`${TAG} L0 recorded: ${filteredMessages.length} messages for session ${sessionKey}`);
|
||||
const maxTs = Math.max(...filteredMessages.map((m) => m.timestamp));
|
||||
return { maxTimestamp: maxTs, messageCount: filteredMessages.length };
|
||||
},
|
||||
);
|
||||
} catch (err) {
|
||||
logger?.error(`${TAG} L0 recording failed: ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
const tL0RecordEnd = performance.now();
|
||||
|
||||
// ============================
|
||||
// Step 1.5: L0 vector indexing — metadata written synchronously,
|
||||
// embedding done in background (non-blocking)
|
||||
// ============================
|
||||
// PERF FIX: Remote embedding API calls (2-3s each) were blocking
|
||||
// the agent_end hook, adding 5-9s latency per conversation round.
|
||||
// Now we:
|
||||
// 1. Write L0 metadata + FTS immediately (no embedding) — ~10ms
|
||||
// 2. Fire off background task to embed + update vectors (non-blocking)
|
||||
// This way the user gets their response immediately.
|
||||
const tL0VecStart = performance.now();
|
||||
let l0VectorsWritten = 0;
|
||||
logger?.debug?.(
|
||||
`${TAG} [L0-vec-index] Check: filteredMessages=${filteredMessages.length}, ` +
|
||||
`vectorStore=${vectorStore ? "available" : "UNAVAILABLE"}, ` +
|
||||
`embeddingService=${embeddingService ? "available" : "UNAVAILABLE"}`,
|
||||
);
|
||||
|
||||
// Pre-generate L0 records and write metadata synchronously (fast path)
|
||||
const l0Records: Array<{ record: L0VectorRecord; content: string }> = [];
|
||||
if (filteredMessages.length > 0 && vectorStore) {
|
||||
const now = new Date().toISOString();
|
||||
logger?.debug?.(`${TAG} [L0-vec-index] START indexing ${filteredMessages.length} message(s) for session ${sessionKey}`);
|
||||
for (let i = 0; i < filteredMessages.length; i++) {
|
||||
const msg = filteredMessages[i];
|
||||
try {
|
||||
const l0Record: L0VectorRecord = {
|
||||
id: generateL0RecordId(sessionKey, i),
|
||||
sessionKey,
|
||||
sessionId: sessionId || "",
|
||||
role: msg.role,
|
||||
messageText: msg.content,
|
||||
recordedAt: now,
|
||||
timestamp: msg.timestamp,
|
||||
};
|
||||
|
||||
// Write metadata + FTS immediately WITHOUT embedding (fast, ~ms)
|
||||
const upsertOk = vectorStore.upsertL0(l0Record, undefined);
|
||||
if (upsertOk) {
|
||||
l0VectorsWritten++;
|
||||
l0Records.push({ record: l0Record, content: msg.content });
|
||||
} else {
|
||||
logger?.warn(`${TAG} [L0-vec-index] upsertL0 returned false for message ${i}`);
|
||||
}
|
||||
} catch (err) {
|
||||
logger?.warn?.(`${TAG} [L0-vec-index] FAILED for message ${i} (non-blocking): ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
}
|
||||
logger?.debug?.(`${TAG} [L0-vec-index] DONE: ${l0VectorsWritten}/${filteredMessages.length} metadata records written (sync)`);
|
||||
|
||||
// Fire-and-forget: batch embed + update vectors in background
|
||||
if (l0Records.length > 0 && embeddingService) {
|
||||
const bgVectorStore = vectorStore; // capture for closure
|
||||
const bgEmbeddingService = embeddingService;
|
||||
const bgRecords = [...l0Records]; // snapshot
|
||||
const bgLogger = logger;
|
||||
|
||||
// Do NOT await — this runs in background after response is sent
|
||||
void (async () => {
|
||||
const tBgStart = performance.now();
|
||||
try {
|
||||
// Use embedBatch for a single API call instead of N sequential calls
|
||||
const texts = bgRecords.map((r) => r.content);
|
||||
const embeddings = await bgEmbeddingService.embedBatch(texts);
|
||||
|
||||
let bgUpdated = 0;
|
||||
for (let i = 0; i < bgRecords.length; i++) {
|
||||
try {
|
||||
const ok = bgVectorStore.updateL0Embedding(bgRecords[i].record.id, embeddings[i]);
|
||||
if (ok) bgUpdated++;
|
||||
} catch (err) {
|
||||
bgLogger?.warn?.(
|
||||
`${TAG} [L0-vec-index-bg] Failed to update embedding for ${bgRecords[i].record.id}: ` +
|
||||
`${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
const bgMs = performance.now() - tBgStart;
|
||||
bgLogger?.debug?.(
|
||||
`${TAG} [L0-vec-index-bg] Background embedding complete: ${bgUpdated}/${bgRecords.length} vectors updated (${bgMs.toFixed(0)}ms)`,
|
||||
);
|
||||
} catch (err) {
|
||||
const bgMs = performance.now() - tBgStart;
|
||||
bgLogger?.warn?.(
|
||||
`${TAG} [L0-vec-index-bg] Background embedding failed (${bgMs.toFixed(0)}ms, non-fatal): ` +
|
||||
`${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
}
|
||||
})();
|
||||
}
|
||||
} else if (filteredMessages.length > 0) {
|
||||
logger?.warn(`${TAG} [L0-vec-index] SKIPPED: vectorStore not available`);
|
||||
}
|
||||
const tL0VecEnd = performance.now();
|
||||
|
||||
// ============================
|
||||
// Step 3: Notify scheduler of this conversation round
|
||||
// ============================
|
||||
const tNotifyStart = performance.now();
|
||||
// Pass empty array: L1 Runner reads from VectorStore DB (or L0 JSONL fallback), not from in-memory buffers.
|
||||
if (scheduler) {
|
||||
await scheduler.notifyConversation(sessionKey, []);
|
||||
logger?.debug?.(`${TAG} Scheduler notified of conversation round (sessionKey=${sessionKey})`);
|
||||
|
||||
const totalMs = performance.now() - tCaptureStart;
|
||||
logger?.info(
|
||||
`${TAG} ⏱ Capture timing: total=${totalMs.toFixed(0)}ms, ` +
|
||||
`l0Record+checkpoint=${(tL0RecordEnd - tL0RecordStart).toFixed(0)}ms, ` +
|
||||
`l0VecIndex=${(tL0VecEnd - tL0VecStart).toFixed(0)}ms ` +
|
||||
`(metadata-only, embed=background, msgs=${filteredMessages.length}), ` +
|
||||
`notify=${(performance.now() - tNotifyStart).toFixed(0)}ms`,
|
||||
);
|
||||
|
||||
return {
|
||||
schedulerNotified: true,
|
||||
l0RecordedCount: filteredMessages.length,
|
||||
l0VectorsWritten,
|
||||
filteredMessages,
|
||||
};
|
||||
}
|
||||
|
||||
const totalMs = performance.now() - tCaptureStart;
|
||||
logger?.info(
|
||||
`${TAG} ⏱ Capture timing: total=${totalMs.toFixed(0)}ms, ` +
|
||||
`l0Record+checkpoint=${(tL0RecordEnd - tL0RecordStart).toFixed(0)}ms, ` +
|
||||
`l0VecIndex=${(tL0VecEnd - tL0VecStart).toFixed(0)}ms ` +
|
||||
`(metadata-only, embed=background, msgs=${filteredMessages.length}), ` +
|
||||
`notify=${(performance.now() - tNotifyStart).toFixed(0)}ms`,
|
||||
);
|
||||
|
||||
logger?.debug?.(`${TAG} No scheduler provided, skipping notification`);
|
||||
return {
|
||||
schedulerNotified: false,
|
||||
l0RecordedCount: filteredMessages.length,
|
||||
l0VectorsWritten,
|
||||
filteredMessages,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,762 @@
|
||||
/**
|
||||
* auto-recall hook (v3): injects relevant memories + persona into agent context
|
||||
* before the agent starts processing.
|
||||
*
|
||||
* - Searches L1 memories using configurable strategy (keyword / embedding / hybrid)
|
||||
* - keyword: FTS5 BM25 (requires FTS5; returns empty if unavailable)
|
||||
* - embedding: VectorStore cosine similarity
|
||||
* - hybrid: keyword + embedding merged with RRF
|
||||
* - L3 persona injection
|
||||
* - L2 scene navigation (full injection, LLM decides relevance)
|
||||
*/
|
||||
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import type { MemoryTdaiConfig } from "../config.js";
|
||||
import { readSceneIndex } from "../scene/scene-index.js";
|
||||
import { generateSceneNavigation, stripSceneNavigation } from "../scene/scene-navigation.js";
|
||||
import type { MemoryRecord } from "../record/l1-reader.js";
|
||||
import type { VectorStore, VectorSearchResult, FtsSearchResult } from "../store/vector-store.js";
|
||||
import { buildFtsQuery } from "../store/vector-store.js";
|
||||
import type { EmbeddingService } from "../store/embedding.js";
|
||||
import { sanitizeText } from "../utils/sanitize.js";
|
||||
|
||||
const TAG = "[memory-tdai] [recall]";
|
||||
|
||||
/**
|
||||
* Memory tools usage guide — injected at the end of memory context so the
|
||||
* main agent knows how to actively retrieve deeper information.
|
||||
*/
|
||||
const MEMORY_TOOLS_GUIDE = `<memory-tools-guide>
|
||||
## 记忆工具调用指南
|
||||
|
||||
当上方注入的记忆片段不足以回答用户问题时，可主动调用以下工具获取更多信息：
|
||||
|
||||
- **tdai_memory_search**：搜索结构化记忆（L1），适用于回忆用户偏好、历史事件节点、规则等关键信息。
|
||||
- **tdai_conversation_search**：搜索原始对话（L0），适用于查找具体消息原文、时间线、上下文细节；也可用于补充或校验 memory_search 的结果。
|
||||
- **read_file**（Scene Navigation 中的路径）：当已定位到相关情境，且需要该场景的完整画像、事件经过或阶段结论时使用。
|
||||
</memory-tools-guide>`
|
||||
|
||||
/**
|
||||
* Build the dynamic scene-navigation read_file hint.
|
||||
* Tells the agent how to resolve relative paths in scene navigation
|
||||
* by prepending the actual pluginDataDir.
|
||||
*/
|
||||
function buildScenePathHint(pluginDataDir: string): string {
|
||||
return `⚠️ Scene Navigation 路径提示：上方 Scene Navigation 中的 Path（如 \`scene_blocks/xxx.md\`）是相对路径，使用 read_file 读取时需拼接为绝对路径：\`${pluginDataDir}/scene_blocks/xxx.md\``;
|
||||
}
|
||||
|
||||
interface Logger {
|
||||
debug?: (message: string) => void;
|
||||
info: (message: string) => void;
|
||||
warn: (message: string) => void;
|
||||
error: (message: string) => void;
|
||||
}
|
||||
|
||||
/** A single recalled L1 memory with its search score and type. */
|
||||
export interface RecalledMemory {
|
||||
content: string;
|
||||
score: number;
|
||||
type: string;
|
||||
}
|
||||
|
||||
export interface RecallResult {
|
||||
/** Injected before user message (prepended to the user's prompt text by openclaw) */
|
||||
prependContext?: string;
|
||||
/** Appended to system prompt (all memory context: persona, scene navigation, relevant memories) */
|
||||
appendSystemContext?: string;
|
||||
|
||||
// ── Metric payload (for pendingRecallCache in index.ts) ──
|
||||
/** L1 memories that were recalled (with scores), for metric reporting */
|
||||
recalledL1Memories?: RecalledMemory[];
|
||||
/** L3 Persona raw content loaded during recall (null if none) */
|
||||
recalledL3Persona?: string | null;
|
||||
/** Effective search strategy used */
|
||||
recallStrategy?: string;
|
||||
}
|
||||
|
||||
export async function performAutoRecall(params: {
|
||||
userText: string;
|
||||
actorId: string;
|
||||
sessionKey: string;
|
||||
cfg: MemoryTdaiConfig;
|
||||
pluginDataDir: string;
|
||||
logger?: Logger;
|
||||
vectorStore?: VectorStore;
|
||||
embeddingService?: EmbeddingService;
|
||||
}): Promise<RecallResult | undefined> {
|
||||
const { cfg, logger } = params;
|
||||
const timeoutMs = cfg.recall.timeoutMs ?? 5000;
|
||||
|
||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||
|
||||
return Promise.race([
|
||||
performAutoRecallInner(params).finally(() => {
|
||||
if (timer) clearTimeout(timer);
|
||||
}),
|
||||
new Promise<undefined>((resolve) => {
|
||||
timer = setTimeout(() => {
|
||||
logger?.warn?.(
|
||||
`${TAG} ⚠️ Recall timed out after ${timeoutMs}ms — skipping memory injection to avoid blocking the user`,
|
||||
);
|
||||
resolve(undefined);
|
||||
}, timeoutMs);
|
||||
}),
|
||||
]);
|
||||
}
|
||||
|
||||
async function performAutoRecallInner(params: {
|
||||
userText: string;
|
||||
actorId: string;
|
||||
sessionKey: string;
|
||||
cfg: MemoryTdaiConfig;
|
||||
pluginDataDir: string;
|
||||
logger?: Logger;
|
||||
vectorStore?: VectorStore;
|
||||
embeddingService?: EmbeddingService;
|
||||
}): Promise<RecallResult | undefined> {
|
||||
const { userText, cfg, pluginDataDir, logger, vectorStore, embeddingService } = params;
|
||||
const tRecallStart = performance.now();
|
||||
|
||||
// Search relevant memories (L1 layer) — skip only when userText is empty/undefined
|
||||
const tSearchStart = performance.now();
|
||||
let memoryLines: string[] = [];
|
||||
let effectiveStrategy = "skipped";
|
||||
let recalledL1Memories: RecalledMemory[] = [];
|
||||
let searchTiming: SearchTiming = { ftsMs: 0, embeddingMs: 0, ftsHits: 0, embeddingHits: 0 };
|
||||
if (!userText || userText.length === 0) {
|
||||
logger?.debug?.(`${TAG} User text empty/undefined, skipping memory search (persona/scene still injected)`);
|
||||
} else {
|
||||
effectiveStrategy = cfg.recall.strategy ?? "hybrid";
|
||||
const searchResult = await searchMemories(userText, pluginDataDir, cfg, logger, effectiveStrategy as "keyword" | "embedding" | "hybrid", vectorStore, embeddingService);
|
||||
memoryLines = searchResult.lines;
|
||||
searchTiming = searchResult.timing;
|
||||
|
||||
// Extract structured RecalledMemory from formatted lines for metric reporting
|
||||
recalledL1Memories = memoryLines.map((line) => {
|
||||
const match = line.match(/^-\s+\[([^\]]+)\]\s+(.+?)(?:\s*\(活动时间:.*\))?$/);
|
||||
if (match) {
|
||||
const tag = match[1];
|
||||
const content = match[2].trim();
|
||||
const typePart = tag.includes("|") ? tag.split("|")[0] : tag;
|
||||
return { content, score: 0, type: typePart };
|
||||
}
|
||||
return { content: line, score: 0, type: "unknown" };
|
||||
});
|
||||
}
|
||||
const tSearchEnd = performance.now();
|
||||
|
||||
// Read persona (L3 layer)
|
||||
const tPersonaStart = performance.now();
|
||||
let personaContent: string | undefined;
|
||||
try {
|
||||
const personaPath = path.join(pluginDataDir, "persona.md");
|
||||
const raw = await fs.readFile(personaPath, "utf-8");
|
||||
personaContent = stripSceneNavigation(raw).trim();
|
||||
if (!personaContent) personaContent = undefined;
|
||||
logger?.debug?.(`${TAG} Persona loaded: ${personaContent ? `${personaContent.length} chars` : "empty"}`);
|
||||
} catch {
|
||||
logger?.debug?.(`${TAG} No persona file found (expected for new users)`);
|
||||
}
|
||||
const tPersonaEnd = performance.now();
|
||||
|
||||
// Load full scene navigation (L2 layer)
|
||||
const tSceneStart = performance.now();
|
||||
let sceneNavigation: string | undefined;
|
||||
try {
|
||||
const sceneIndex = await readSceneIndex(pluginDataDir);
|
||||
if (sceneIndex.length > 0) {
|
||||
sceneNavigation = generateSceneNavigation(sceneIndex);
|
||||
logger?.debug?.(`${TAG} Scene navigation generated: ${sceneIndex.length} scenes`);
|
||||
}
|
||||
} catch {
|
||||
logger?.debug?.(`${TAG} No scene index found`);
|
||||
}
|
||||
const tSceneEnd = performance.now();
|
||||
|
||||
if (memoryLines.length === 0 && !personaContent && !sceneNavigation) {
|
||||
const totalMs = performance.now() - tRecallStart;
|
||||
logger?.info(
|
||||
`${TAG} ⏱ Recall timing: total=${totalMs.toFixed(0)}ms, ` +
|
||||
`search=${(tSearchEnd - tSearchStart).toFixed(0)}ms(strategy=${effectiveStrategy},hits=${memoryLines.length},` +
|
||||
`fts=${searchTiming.ftsMs.toFixed(0)}ms/${searchTiming.ftsHits}hits,` +
|
||||
`vec=${searchTiming.embeddingMs.toFixed(0)}ms/${searchTiming.embeddingHits}hits), ` +
|
||||
`persona=${(tPersonaEnd - tPersonaStart).toFixed(0)}ms, ` +
|
||||
`scene=${(tSceneEnd - tSceneStart).toFixed(0)}ms — no context to inject`,
|
||||
);
|
||||
logger?.debug?.(`${TAG} No memories/persona/scenes to inject`);
|
||||
return undefined;
|
||||
}
|
||||
|
||||
// All memory context → appendSystemContext (system prompt end)
|
||||
// Order: user-persona → scene-navigation → relevant-memories → tools-guide
|
||||
const systemParts: string[] = [];
|
||||
if (personaContent) {
|
||||
systemParts.push(`<user-persona>\n${personaContent}\n</user-persona>`);
|
||||
}
|
||||
if (sceneNavigation) {
|
||||
const pathHint = buildScenePathHint(pluginDataDir);
|
||||
systemParts.push(`<scene-navigation>\n${sceneNavigation}\n\n${pathHint}\n</scene-navigation>`);
|
||||
}
|
||||
if (memoryLines.length > 0) {
|
||||
systemParts.push(`<relevant-memories>\n${memoryLines.join("\n")}\n</relevant-memories>`);
|
||||
}
|
||||
|
||||
// Append memory tools usage guide so the agent knows how to actively
|
||||
// retrieve deeper context when the injected snippets are not enough.
|
||||
if (systemParts.length > 0) {
|
||||
systemParts.push(MEMORY_TOOLS_GUIDE);
|
||||
}
|
||||
|
||||
const appendSystemContext = systemParts.length > 0 ? systemParts.join("\n\n") : undefined;
|
||||
|
||||
const totalMs = performance.now() - tRecallStart;
|
||||
logger?.info(
|
||||
`${TAG} ⏱ Recall timing: total=${totalMs.toFixed(0)}ms, ` +
|
||||
`search=${(tSearchEnd - tSearchStart).toFixed(0)}ms(strategy=${effectiveStrategy},hits=${memoryLines.length},` +
|
||||
`fts=${searchTiming.ftsMs.toFixed(0)}ms/${searchTiming.ftsHits}hits,` +
|
||||
`vec=${searchTiming.embeddingMs.toFixed(0)}ms/${searchTiming.embeddingHits}hits), ` +
|
||||
`persona=${(tPersonaEnd - tPersonaStart).toFixed(0)}ms(${personaContent ? `${personaContent.length}chars` : "none"}), ` +
|
||||
`scene=${(tSceneEnd - tSceneStart).toFixed(0)}ms(${sceneNavigation ? "loaded" : "none"})`,
|
||||
);
|
||||
|
||||
if (!appendSystemContext) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return {
|
||||
appendSystemContext,
|
||||
recalledL1Memories,
|
||||
recalledL3Persona: personaContent ?? null,
|
||||
recallStrategy: effectiveStrategy,
|
||||
};
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Multi-strategy search dispatcher
|
||||
// ============================
|
||||
|
||||
interface ScoredRecord {
|
||||
record: MemoryRecord;
|
||||
score: number;
|
||||
}
|
||||
|
||||
/** Timing breakdown from memory search */
|
||||
interface SearchTiming {
|
||||
ftsMs: number;
|
||||
embeddingMs: number;
|
||||
ftsHits: number;
|
||||
embeddingHits: number;
|
||||
}
|
||||
|
||||
interface SearchResult {
|
||||
lines: string[];
|
||||
timing: SearchTiming;
|
||||
}
|
||||
|
||||
/**
|
||||
* Search memories and return both formatted lines and structured details.
|
||||
*
|
||||
* This is a thin wrapper around `searchMemories` that also captures
|
||||
* the recalled memory metadata for metric reporting (agent_turn event).
|
||||
* It parses the returned formatted lines to extract type/content info.
|
||||
*/
|
||||
async function searchMemoriesWithDetails(
|
||||
userText: string,
|
||||
pluginDataDir: string,
|
||||
cfg: MemoryTdaiConfig,
|
||||
logger: Logger | undefined,
|
||||
strategy: "keyword" | "embedding" | "hybrid",
|
||||
vectorStore?: VectorStore,
|
||||
embeddingService?: EmbeddingService,
|
||||
): Promise<{ lines: string[]; memories: RecalledMemory[]; timing: SearchTiming }> {
|
||||
const result = await searchMemories(userText, pluginDataDir, cfg, logger, strategy, vectorStore, embeddingService);
|
||||
|
||||
// Extract structured data from formatted memory lines.
|
||||
// Format: "- [type|scene] content (活动时间: ...)" or "- [type] content"
|
||||
const memories: RecalledMemory[] = result.lines.map((line) => {
|
||||
const match = line.match(/^-\s+\[([^\]]+)\]\s+(.+?)(?:\s*\(活动时间:.*\))?$/);
|
||||
if (match) {
|
||||
const tag = match[1];
|
||||
const content = match[2].trim();
|
||||
const typePart = tag.includes("|") ? tag.split("|")[0] : tag;
|
||||
return { content, score: 0, type: typePart };
|
||||
}
|
||||
return { content: line, score: 0, type: "unknown" };
|
||||
});
|
||||
|
||||
return { lines: result.lines, memories, timing: result.timing };
|
||||
}
|
||||
|
||||
/**
|
||||
* Search memories using the configured strategy.
|
||||
*
|
||||
* - "keyword": JSONL keyword-based (Jaccard similarity) — no embedding needed
|
||||
* - "embedding": VectorStore cosine similarity — requires vectorStore + embeddingService
|
||||
* - "hybrid": merge both keyword and embedding results with RRF (Reciprocal Rank Fusion)
|
||||
*
|
||||
* Falls back to keyword if embedding resources are unavailable.
|
||||
*/
|
||||
async function searchMemories(
|
||||
userText: string,
|
||||
pluginDataDir: string,
|
||||
cfg: MemoryTdaiConfig,
|
||||
logger: Logger | undefined,
|
||||
strategy: "keyword" | "embedding" | "hybrid",
|
||||
vectorStore?: VectorStore,
|
||||
embeddingService?: EmbeddingService,
|
||||
): Promise<SearchResult> {
|
||||
const emptyResult: SearchResult = { lines: [], timing: { ftsMs: 0, embeddingMs: 0, ftsHits: 0, embeddingHits: 0 } };
|
||||
// Strip gateway-injected inbound metadata (Sender, timestamps, media markers,
|
||||
// base64 image data, etc.) so FTS / embedding queries are based on pure user intent.
|
||||
const cleanText = sanitizeText(userText);
|
||||
|
||||
if (cleanText.length < 2) {
|
||||
logger?.debug?.(`${TAG} Query too short for memory search (raw=${userText.length}, clean=${cleanText.length})`);
|
||||
return emptyResult;
|
||||
}
|
||||
|
||||
if (cleanText.length !== userText.length) {
|
||||
logger?.debug?.(
|
||||
`${TAG} userText sanitized: ${userText.length} → ${cleanText.length} chars`,
|
||||
);
|
||||
}
|
||||
|
||||
const maxResults = cfg.recall.maxResults ?? 5;
|
||||
const threshold = cfg.recall.scoreThreshold ?? 0.3;
|
||||
|
||||
const embeddingAvailable = !!vectorStore && !!embeddingService;
|
||||
|
||||
logger?.debug?.(
|
||||
`${TAG} [searchMemories] strategy=${strategy}, embeddingAvailable=${embeddingAvailable}, ` +
|
||||
`vectorStore=${vectorStore ? "available" : "UNAVAILABLE"}, ` +
|
||||
`embeddingService=${embeddingService ? "available" : "UNAVAILABLE"}, ` +
|
||||
`maxResults=${maxResults}, threshold=${threshold}`,
|
||||
);
|
||||
|
||||
// Determine effective strategy (fall back to keyword if embedding not available)
|
||||
let effectiveStrategy = strategy;
|
||||
if ((strategy === "embedding" || strategy === "hybrid") && !embeddingAvailable) {
|
||||
logger?.warn?.(
|
||||
`${TAG} Strategy "${strategy}" requested but EmbeddingService not available, falling back to keyword`,
|
||||
);
|
||||
effectiveStrategy = "keyword";
|
||||
}
|
||||
|
||||
logger?.debug?.(`${TAG} Search strategy: ${effectiveStrategy} (configured: ${strategy})`);
|
||||
|
||||
try {
|
||||
if (effectiveStrategy === "keyword") {
|
||||
const tFts = performance.now();
|
||||
const lines = await searchByKeyword(cleanText, pluginDataDir, maxResults, threshold, logger, vectorStore);
|
||||
return { lines, timing: { ftsMs: performance.now() - tFts, embeddingMs: 0, ftsHits: lines.length, embeddingHits: 0 } };
|
||||
}
|
||||
|
||||
if (effectiveStrategy === "embedding") {
|
||||
const tEmb = performance.now();
|
||||
const lines = await searchByEmbedding(cleanText, maxResults, threshold, vectorStore!, embeddingService!, logger);
|
||||
return { lines, timing: { ftsMs: 0, embeddingMs: performance.now() - tEmb, ftsHits: 0, embeddingHits: lines.length } };
|
||||
}
|
||||
|
||||
// Hybrid: run both keyword and embedding, merge with RRF
|
||||
return await searchHybrid(cleanText, pluginDataDir, maxResults, threshold, vectorStore!, embeddingService!, logger);
|
||||
} catch (err) {
|
||||
logger?.warn?.(`${TAG} Memory search failed (strategy=${effectiveStrategy}): ${err instanceof Error ? err.message : String(err)}`);
|
||||
return emptyResult;
|
||||
}
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Strategy: Keyword (FTS5 BM25, no in-memory fallback)
|
||||
// ============================
|
||||
|
||||
async function searchByKeyword(
|
||||
userText: string,
|
||||
_pluginDataDir: string,
|
||||
maxResults: number,
|
||||
threshold: number,
|
||||
logger?: Logger,
|
||||
vectorStore?: VectorStore,
|
||||
): Promise<string[]> {
|
||||
// Prefer FTS5 if available
|
||||
if (vectorStore?.isFtsAvailable()) {
|
||||
const ftsQuery = buildFtsQuery(userText);
|
||||
if (ftsQuery) {
|
||||
logger?.debug?.(`${TAG} [keyword-fts] Using FTS5 BM25 search: query="${ftsQuery}"`);
|
||||
const ftsResults = vectorStore.ftsSearchL1(ftsQuery, maxResults * 2);
|
||||
if (ftsResults.length > 0) {
|
||||
logger?.debug?.(
|
||||
`${TAG} [keyword-fts] FTS5 raw results (${ftsResults.length}): ` +
|
||||
ftsResults.map((r) => `id=${r.record_id} score=${r.score.toFixed(6)}`).join(", "),
|
||||
);
|
||||
const filtered = ftsResults
|
||||
.filter((r) => r.score >= threshold)
|
||||
.slice(0, maxResults);
|
||||
|
||||
if (filtered.length > 0) {
|
||||
logger?.debug?.(`${TAG} [keyword-fts] FTS5 found ${filtered.length} results (from ${ftsResults.length} raw, threshold=${threshold})`);
|
||||
return filtered.map((r) => formatMemoryLine(ftsResultToFormatable(r)));
|
||||
}
|
||||
|
||||
// BM25 absolute scores are unreliable when the document set is very
|
||||
// small (e.g. 1–3 records) because IDF approaches 0. In that case,
|
||||
// trust FTS5's MATCH + rank ordering and return the top results anyway.
|
||||
if (ftsResults.length <= maxResults) {
|
||||
logger?.debug?.(
|
||||
`${TAG} [keyword-fts] All ${ftsResults.length} results below threshold=${threshold} ` +
|
||||
`but document set is small — returning all matched results`,
|
||||
);
|
||||
return ftsResults.slice(0, maxResults).map((r) => formatMemoryLine(ftsResultToFormatable(r)));
|
||||
}
|
||||
logger?.debug?.(`${TAG} [keyword-fts] FTS5 returned 0 results above threshold (from ${ftsResults.length} raw)`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// FTS5 not available or returned no results — skip in-memory fallback to avoid O(N) full scan
|
||||
logger?.debug?.(`${TAG} [keyword] FTS5 unavailable or no results, skipping keyword search`);
|
||||
return [];
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Strategy: Embedding (VectorStore cosine)
|
||||
// ============================
|
||||
|
||||
async function searchByEmbedding(
|
||||
userText: string,
|
||||
maxResults: number,
|
||||
threshold: number,
|
||||
vectorStore: VectorStore,
|
||||
embeddingService: EmbeddingService,
|
||||
logger?: Logger,
|
||||
): Promise<string[]> {
|
||||
logger?.debug?.(
|
||||
`${TAG} [embedding-search] START query="${userText.slice(0, 80)}...", maxResults=${maxResults}, threshold=${threshold}`,
|
||||
);
|
||||
const queryEmbedding = await embeddingService.embed(userText);
|
||||
logger?.debug?.(
|
||||
`${TAG} [embedding-search] Query embedding OK: dims=${queryEmbedding.length}, ` +
|
||||
`norm=${Math.sqrt(Array.from(queryEmbedding).reduce((s, v) => s + v * v, 0)).toFixed(4)}, ` +
|
||||
`searching top-${maxResults * 2}...`,
|
||||
);
|
||||
// Retrieve more candidates for subsequent filtering
|
||||
const vecResults: VectorSearchResult[] = vectorStore.search(queryEmbedding, maxResults * 2);
|
||||
|
||||
if (vecResults.length === 0) {
|
||||
logger?.debug?.(`${TAG} [embedding-search] Returned 0 results`);
|
||||
return [];
|
||||
}
|
||||
|
||||
logger?.debug?.(`${TAG} [embedding-search] Got ${vecResults.length} candidates, filtering by threshold=${threshold}`);
|
||||
for (const r of vecResults) {
|
||||
logger?.debug?.(
|
||||
`${TAG} [embedding-search] candidate id=${r.record_id}, score=${r.score.toFixed(4)}, ` +
|
||||
`type=${r.type}, content="${r.content.slice(0, 60)}..."`,
|
||||
);
|
||||
}
|
||||
|
||||
const filtered = vecResults
|
||||
.filter((r) => r.score >= threshold)
|
||||
.slice(0, maxResults);
|
||||
|
||||
if (filtered.length > 0) {
|
||||
logger?.debug?.(`${TAG} [embedding-search] Found ${filtered.length} relevant memories above threshold (from ${vecResults.length} candidates)`);
|
||||
return filtered.map((r) => formatMemoryLine(vectorResultToFormatable(r)));
|
||||
}
|
||||
|
||||
logger?.debug?.(`${TAG} [embedding-search] No results above threshold ${threshold}`);
|
||||
return [];
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Strategy: Hybrid (Keyword + Embedding + RRF)
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Hybrid search: run keyword (FTS5) and embedding in parallel, merge with
|
||||
* Reciprocal Rank Fusion (RRF) to combine rank lists.
|
||||
*
|
||||
* RRF score for a record at rank r = 1 / (k + r), where k=60 is a constant.
|
||||
* If a record appears in both lists, its RRF scores are summed.
|
||||
*
|
||||
* If FTS5 is unavailable, the keyword side returns empty and RRF uses
|
||||
* embedding results only.
|
||||
*/
|
||||
async function searchHybrid(
|
||||
userText: string,
|
||||
_pluginDataDir: string,
|
||||
maxResults: number,
|
||||
_threshold: number,
|
||||
vectorStore: VectorStore,
|
||||
embeddingService: EmbeddingService,
|
||||
logger?: Logger,
|
||||
): Promise<SearchResult> {
|
||||
// Run keyword and embedding searches in parallel
|
||||
const candidateK = maxResults * 3; // retrieve more for merging
|
||||
|
||||
const [keywordResult, embeddingResult] = await Promise.all([
|
||||
// Keyword search: FTS5 only (no in-memory fallback)
|
||||
(async () => {
|
||||
const tStart = performance.now();
|
||||
try {
|
||||
// Try FTS5 first
|
||||
if (vectorStore.isFtsAvailable()) {
|
||||
const ftsQuery = buildFtsQuery(userText);
|
||||
if (ftsQuery) {
|
||||
const ftsResults = vectorStore.ftsSearchL1(ftsQuery, candidateK);
|
||||
if (ftsResults.length > 0) {
|
||||
logger?.debug?.(`${TAG} [hybrid-keyword-fts] FTS5 found ${ftsResults.length} candidates`);
|
||||
// Convert FtsSearchResult to ScoredRecord for RRF merge
|
||||
const records = ftsResults.map((r): ScoredRecord => ({
|
||||
record: {
|
||||
id: r.record_id,
|
||||
content: r.content,
|
||||
type: r.type as MemoryRecord["type"],
|
||||
priority: r.priority,
|
||||
scene_name: r.scene_name,
|
||||
source_message_ids: [],
|
||||
metadata: r.metadata_json ? (() => { try { return JSON.parse(r.metadata_json); } catch { return {}; } })() : {},
|
||||
timestamps: [r.timestamp_str].filter(Boolean),
|
||||
createdAt: "",
|
||||
updatedAt: "",
|
||||
sessionKey: r.session_key,
|
||||
sessionId: r.session_id,
|
||||
},
|
||||
score: r.score,
|
||||
}));
|
||||
return { records, ms: performance.now() - tStart };
|
||||
}
|
||||
}
|
||||
}
|
||||
// FTS5 not available or returned no results — skip in-memory fallback
|
||||
logger?.debug?.(`${TAG} [hybrid-keyword] FTS5 unavailable or no results, skipping keyword part`);
|
||||
return { records: [] as ScoredRecord[], ms: performance.now() - tStart };
|
||||
} catch (err) {
|
||||
logger?.warn?.(`${TAG} Hybrid: keyword part failed: ${err instanceof Error ? err.message : String(err)}`);
|
||||
return { records: [] as ScoredRecord[], ms: performance.now() - tStart };
|
||||
}
|
||||
})(),
|
||||
// Embedding search
|
||||
(async () => {
|
||||
const tStart = performance.now();
|
||||
try {
|
||||
logger?.debug?.(`${TAG} [hybrid-embedding] Generating query embedding...`);
|
||||
const queryEmbedding = await embeddingService.embed(userText);
|
||||
logger?.debug?.(
|
||||
`${TAG} [hybrid-embedding] Embedding OK, dims=${queryEmbedding.length}, searching top-${candidateK}...`,
|
||||
);
|
||||
const results = vectorStore.search(queryEmbedding, candidateK);
|
||||
logger?.debug?.(`${TAG} [hybrid-embedding] Got ${results.length} candidates`);
|
||||
return { results, ms: performance.now() - tStart };
|
||||
} catch (err) {
|
||||
logger?.warn?.(`${TAG} Hybrid: embedding part failed: ${err instanceof Error ? err.message : String(err)}`);
|
||||
return { results: [] as VectorSearchResult[], ms: performance.now() - tStart };
|
||||
}
|
||||
})(),
|
||||
]);
|
||||
|
||||
const keywordResults = keywordResult.records;
|
||||
const embeddingResults = embeddingResult.results;
|
||||
const timing: SearchTiming = {
|
||||
ftsMs: keywordResult.ms,
|
||||
embeddingMs: embeddingResult.ms,
|
||||
ftsHits: keywordResults.length,
|
||||
embeddingHits: embeddingResults.length,
|
||||
};
|
||||
|
||||
if (keywordResults.length === 0 && embeddingResults.length === 0) {
|
||||
logger?.debug?.(`${TAG} Hybrid search: both strategies returned 0 results`);
|
||||
return { lines: [], timing };
|
||||
}
|
||||
|
||||
// RRF merge: k=60 is a standard constant from the RRF paper
|
||||
const RRF_K = 60;
|
||||
|
||||
// Map: record_id → { rrfScore, formatable }
|
||||
const mergedMap = new Map<string, { rrfScore: number; formatable: FormatableMemory }>();
|
||||
|
||||
// Process keyword results
|
||||
for (let rank = 0; rank < keywordResults.length; rank++) {
|
||||
const r = keywordResults[rank];
|
||||
const id = r.record.id;
|
||||
const rrfScore = 1 / (RRF_K + rank + 1);
|
||||
const existing = mergedMap.get(id);
|
||||
if (existing) {
|
||||
existing.rrfScore += rrfScore;
|
||||
} else {
|
||||
mergedMap.set(id, { rrfScore, formatable: recordToFormatable(r.record) });
|
||||
}
|
||||
}
|
||||
|
||||
// Process embedding results
|
||||
for (let rank = 0; rank < embeddingResults.length; rank++) {
|
||||
const r = embeddingResults[rank];
|
||||
const id = r.record_id;
|
||||
const rrfScore = 1 / (RRF_K + rank + 1);
|
||||
const existing = mergedMap.get(id);
|
||||
if (existing) {
|
||||
existing.rrfScore += rrfScore;
|
||||
} else {
|
||||
mergedMap.set(id, { rrfScore, formatable: vectorResultToFormatable(r) });
|
||||
}
|
||||
}
|
||||
|
||||
// Sort by combined RRF score and take top results
|
||||
const sorted = [...mergedMap.entries()]
|
||||
.sort((a, b) => b[1].rrfScore - a[1].rrfScore)
|
||||
.slice(0, maxResults);
|
||||
|
||||
if (sorted.length > 0) {
|
||||
logger?.debug?.(
|
||||
`${TAG} Hybrid search found ${sorted.length} results ` +
|
||||
`(keyword=${keywordResults.length}, embedding=${embeddingResults.length})`,
|
||||
);
|
||||
return { lines: sorted.map(([, { formatable }]) => formatMemoryLine(formatable)), timing };
|
||||
}
|
||||
|
||||
logger?.debug?.(`${TAG} Hybrid search: no results after merge`);
|
||||
return { lines: [], timing };
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Unified memory line formatter
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Format a single memory record into a rich natural-language line for prompt injection.
|
||||
*
|
||||
* Time semantics:
|
||||
* - timestamp (点时间): when the activity/event happened, e.g. "2025-03-01 mentioned something"
|
||||
* - activity_start_time / activity_end_time (段时间): activity time range, e.g. "trip from 2025-05-01 to 2025-05-10"
|
||||
* - All three time fields may be empty/undefined — handled gracefully.
|
||||
*
|
||||
* Output examples:
|
||||
* - [persona] 用户叫王小明，30岁，是一名软件工程师。
|
||||
* - [episodic|旅行计划] 用户计划五月去日本旅行。(活动时间: 2025-05-01 ~ 2025-05-10)
|
||||
* - [episodic] 用户今天加班到很晚。(活动时间: 2025-03-01)
|
||||
* - [instruction] 用户要求回答时使用中文，保持简洁。
|
||||
*/
|
||||
interface FormatableMemory {
|
||||
type: string;
|
||||
content: string;
|
||||
scene_name?: string;
|
||||
/** Activity time range start (段时间 start), may be empty */
|
||||
activity_start_time?: string;
|
||||
/** Activity time range end (段时间 end), may be empty */
|
||||
activity_end_time?: string;
|
||||
/** Activity point-in-time (点时间: when it happened), may be empty */
|
||||
timestamp?: string;
|
||||
}
|
||||
|
||||
function formatMemoryLine(m: FormatableMemory): string {
|
||||
// 1. Type tag + optional scene name
|
||||
const tag = m.scene_name ? `${m.type}|${m.scene_name}` : m.type;
|
||||
|
||||
// 2. Content (core)
|
||||
let line = `- [${tag}] ${m.content}`;
|
||||
|
||||
// 3. Time info — prefer activity_start/end range; fall back to timestamp as point-in-time
|
||||
const start = formatTimestamp(m.activity_start_time);
|
||||
const end = formatTimestamp(m.activity_end_time);
|
||||
const point = formatTimestamp(m.timestamp);
|
||||
|
||||
if (start && end) {
|
||||
// 段时间: both start and end
|
||||
line += ` (活动时间: ${start} ~ ${end})`;
|
||||
} else if (start) {
|
||||
// 段时间: only start
|
||||
line += ` (活动时间: ${start}起)`;
|
||||
} else if (end) {
|
||||
// 段时间: only end
|
||||
line += ` (活动时间: 至${end})`;
|
||||
} else if (point) {
|
||||
// 点时间: single timestamp
|
||||
line += ` (活动时间: ${point})`;
|
||||
}
|
||||
// If all three are empty → no time info appended (graceful)
|
||||
|
||||
return line;
|
||||
}
|
||||
|
||||
/**
|
||||
* Format an ISO 8601 timestamp to a concise date or datetime string.
|
||||
* - If the time part is 00:00:00 → show date only (e.g. "2025-03-01")
|
||||
* - Otherwise → show date + time (e.g. "2025-03-01 14:30")
|
||||
* - Returns undefined for empty/invalid inputs.
|
||||
*/
|
||||
function formatTimestamp(ts: string | undefined): string | undefined {
|
||||
if (!ts) return undefined;
|
||||
// Try to parse ISO format: "2025-03-01T14:30:00.000Z" or "2025-03-01"
|
||||
const match = ts.match(/^(\d{4}-\d{2}-\d{2})(?:T(\d{2}:\d{2})(?::\d{2})?)?/);
|
||||
if (!match) return undefined;
|
||||
const datePart = match[1];
|
||||
const timePart = match[2];
|
||||
if (!timePart || timePart === "00:00") {
|
||||
return datePart;
|
||||
}
|
||||
return `${datePart} ${timePart}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a FormatableMemory from a full MemoryRecord (keyword search path).
|
||||
* Handles empty metadata, empty timestamps array gracefully.
|
||||
*/
|
||||
function recordToFormatable(record: MemoryRecord): FormatableMemory {
|
||||
const meta = record.metadata as { activity_start_time?: string; activity_end_time?: string } | undefined;
|
||||
return {
|
||||
type: record.type,
|
||||
content: record.content,
|
||||
scene_name: record.scene_name || undefined,
|
||||
activity_start_time: meta?.activity_start_time || undefined,
|
||||
activity_end_time: meta?.activity_end_time || undefined,
|
||||
timestamp: (record.timestamps && record.timestamps.length > 0) ? record.timestamps[0] : undefined,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a FormatableMemory from a VectorSearchResult (embedding search path).
|
||||
* Handles empty/invalid metadata_json, empty timestamp_str gracefully.
|
||||
*/
|
||||
function vectorResultToFormatable(r: VectorSearchResult): FormatableMemory {
|
||||
let activityStart: string | undefined;
|
||||
let activityEnd: string | undefined;
|
||||
if (r.metadata_json && r.metadata_json !== "{}") {
|
||||
try {
|
||||
const meta = typeof r.metadata_json === "string" ? JSON.parse(r.metadata_json) : r.metadata_json;
|
||||
activityStart = meta?.activity_start_time || undefined;
|
||||
activityEnd = meta?.activity_end_time || undefined;
|
||||
} catch { /* ignore parse errors — treat as no metadata */ }
|
||||
}
|
||||
return {
|
||||
type: r.type,
|
||||
content: r.content,
|
||||
scene_name: r.scene_name || undefined,
|
||||
activity_start_time: activityStart,
|
||||
activity_end_time: activityEnd,
|
||||
timestamp: r.timestamp_str || undefined,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a FormatableMemory from an FtsSearchResult (FTS5 keyword search path).
|
||||
* Handles empty/invalid metadata_json, empty timestamp_str gracefully.
|
||||
*/
|
||||
function ftsResultToFormatable(r: FtsSearchResult): FormatableMemory {
|
||||
let activityStart: string | undefined;
|
||||
let activityEnd: string | undefined;
|
||||
if (r.metadata_json && r.metadata_json !== "{}") {
|
||||
try {
|
||||
const meta = typeof r.metadata_json === "string" ? JSON.parse(r.metadata_json) : r.metadata_json;
|
||||
activityStart = meta?.activity_start_time || undefined;
|
||||
activityEnd = meta?.activity_end_time || undefined;
|
||||
} catch { /* ignore parse errors — treat as no metadata */ }
|
||||
}
|
||||
return {
|
||||
type: r.type,
|
||||
content: r.content,
|
||||
scene_name: r.scene_name || undefined,
|
||||
activity_start_time: activityStart,
|
||||
activity_end_time: activityEnd,
|
||||
timestamp: r.timestamp_str || undefined,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,205 @@
|
||||
/**
|
||||
* PersonaGenerator: generates or updates user persona using the four-layer
|
||||
* deep scan model via CleanContextRunner.
|
||||
*/
|
||||
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { CleanContextRunner } from "../utils/clean-context-runner.js";
|
||||
import { CheckpointManager } from "../utils/checkpoint.js";
|
||||
import { readSceneIndex } from "../scene/scene-index.js";
|
||||
import { generateSceneNavigation, stripSceneNavigation } from "../scene/scene-navigation.js";
|
||||
import { buildPersonaPrompt } from "../prompts/persona-generation.js";
|
||||
import { BackupManager } from "../utils/backup.js";
|
||||
import { escapeXmlTags } from "../utils/sanitize.js";
|
||||
import { report } from "../report/reporter.js";
|
||||
|
||||
const TAG = "[memory-tdai] [persona]";
|
||||
|
||||
interface Logger {
|
||||
debug?: (message: string) => void;
|
||||
info: (message: string) => void;
|
||||
warn: (message: string) => void;
|
||||
error: (message: string) => void;
|
||||
}
|
||||
|
||||
export class PersonaGenerator {
|
||||
private dataDir: string;
|
||||
private runner: CleanContextRunner;
|
||||
private logger: Logger | undefined;
|
||||
private backupCount: number;
|
||||
private instanceId: string | undefined;
|
||||
|
||||
constructor(opts: {
|
||||
dataDir: string;
|
||||
config: unknown;
|
||||
model?: string;
|
||||
backupCount?: number;
|
||||
logger?: Logger;
|
||||
/** Plugin instance ID for metric reporting (optional) */
|
||||
instanceId?: string;
|
||||
}) {
|
||||
this.dataDir = opts.dataDir;
|
||||
this.logger = opts.logger;
|
||||
this.backupCount = opts.backupCount ?? 3;
|
||||
this.instanceId = opts.instanceId;
|
||||
this.runner = new CleanContextRunner({
|
||||
config: opts.config,
|
||||
modelRef: opts.model,
|
||||
enableTools: true,
|
||||
logger: opts.logger,
|
||||
});
|
||||
this.logger?.debug?.(`${TAG} Generator created: model=${opts.model ?? "(default)"}, dataDir=${opts.dataDir}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute persona generation.
|
||||
*/
|
||||
async generate(triggerReason?: string): Promise<boolean> {
|
||||
const startMs = Date.now();
|
||||
this.logger?.debug?.(`${TAG} Starting generation: reason="${triggerReason ?? "none"}"`);
|
||||
|
||||
const cpManager = new CheckpointManager(this.dataDir);
|
||||
const cp = await cpManager.read();
|
||||
this.logger?.debug?.(`${TAG} Checkpoint: total_processed=${cp.total_processed}, last_persona_at=${cp.last_persona_at}`);
|
||||
|
||||
const personaPath = path.join(this.dataDir, "persona.md");
|
||||
|
||||
// 1. Read existing persona (strip navigation)
|
||||
let existingPersona: string | undefined;
|
||||
try {
|
||||
const raw = await fs.readFile(personaPath, "utf-8");
|
||||
existingPersona = stripSceneNavigation(raw).trim() || undefined;
|
||||
this.logger?.debug?.(`${TAG} Existing persona: ${existingPersona ? `${existingPersona.length} chars` : "empty"}`);
|
||||
} catch {
|
||||
this.logger?.debug?.(`${TAG} No existing persona file`);
|
||||
}
|
||||
|
||||
// 2. Load scene index + identify changed scenes
|
||||
const index = await readSceneIndex(this.dataDir);
|
||||
const changedScenes = index.filter((e) => {
|
||||
if (!cp.last_persona_time) return true;
|
||||
const updatedMs = new Date(e.updated).getTime();
|
||||
const personaMs = new Date(cp.last_persona_time).getTime();
|
||||
// If either date is unparseable (NaN), treat as changed (conservative)
|
||||
if (Number.isNaN(updatedMs) || Number.isNaN(personaMs)) return true;
|
||||
return updatedMs > personaMs;
|
||||
});
|
||||
this.logger?.debug?.(`${TAG} Scene index: ${index.length} total, ${changedScenes.length} changed since last persona`);
|
||||
|
||||
// 3. Read changed scene contents (full raw content including META, matching Python reference)
|
||||
const blocksDir = path.join(this.dataDir, "scene_blocks");
|
||||
const changedSceneContents: string[] = [];
|
||||
for (const entry of changedScenes) {
|
||||
try {
|
||||
const raw = await fs.readFile(path.join(blocksDir, entry.filename), "utf-8");
|
||||
changedSceneContents.push(
|
||||
`### [${changedSceneContents.length + 1}] ${entry.filename}\n\n\`\`\`markdown\n${raw}\n\`\`\``,
|
||||
);
|
||||
} catch {
|
||||
this.logger?.warn(`${TAG} Could not read scene block: ${entry.filename}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (changedSceneContents.length === 0 && existingPersona) {
|
||||
this.logger?.debug?.(`${TAG} No scene changes and persona exists, skipping generation`);
|
||||
return false;
|
||||
}
|
||||
|
||||
// 4. Determine mode
|
||||
const mode = existingPersona ? "incremental" : "first";
|
||||
this.logger?.debug?.(`${TAG} Generation mode: ${mode}, ${changedSceneContents.length} scene blocks to process`);
|
||||
|
||||
// 5. Build changed scenes section with guidance (matching Python reference format)
|
||||
let changedScenesContent: string;
|
||||
if (changedSceneContents.length > 0) {
|
||||
changedScenesContent =
|
||||
`\n\n## 📄 变化场景完整内容\n\n` +
|
||||
`*自上次 Persona 更新后，以下 ${changedSceneContents.length} 个场景发生了变化。工程已为你预加载完整内容：*\n\n` +
|
||||
changedSceneContents.join("\n\n") +
|
||||
`\n\n---\n\n` +
|
||||
`⚠️ **重点分析变化场景**：上述场景是自上次更新后的**新增/修改内容**，请**重点分析**这些场景中的新信息。\n`;
|
||||
} else {
|
||||
changedScenesContent = `\n\n⚠️ **无变化场景**：所有场景均已在上次 Persona 更新中分析过，本次可直接读取所有场景进行全局审视。\n`;
|
||||
}
|
||||
|
||||
// 6. Build prompt
|
||||
const prompt = buildPersonaPrompt({
|
||||
mode,
|
||||
currentTime: new Date().toISOString(),
|
||||
totalProcessed: cp.total_processed,
|
||||
sceneCount: index.length,
|
||||
changedSceneCount: changedScenes.length,
|
||||
changedScenesContent,
|
||||
existingPersona,
|
||||
triggerInfo: triggerReason,
|
||||
personaFilePath: personaPath,
|
||||
checkpointPath: path.join(this.dataDir, ".metadata", "recall_checkpoint.json"),
|
||||
});
|
||||
|
||||
// 7. Backup before LLM run (LLM writes persona.md via tools)
|
||||
const bm = new BackupManager(path.join(this.dataDir, ".backup"));
|
||||
await bm.backupFile(personaPath, "persona", `offset${cp.total_processed}`, this.backupCount);
|
||||
|
||||
// 8. Run LLM agent (sandboxed to dataDir, tools enabled — LLM writes persona.md directly)
|
||||
try {
|
||||
this.logger?.debug?.(`${TAG} Calling LLM for persona generation (timeout=180s, tools=enabled, workspaceDir=${this.dataDir})...`);
|
||||
await this.runner.run({
|
||||
prompt,
|
||||
taskId: "persona-generation",
|
||||
timeoutMs: 180_000,
|
||||
// maxTokens omitted → core uses the resolved model's maxTokens from catalog
|
||||
workspaceDir: this.dataDir,
|
||||
});
|
||||
this.logger?.debug?.(`${TAG} LLM runner completed`);
|
||||
} catch (err) {
|
||||
const elapsedMs = Date.now() - startMs;
|
||||
this.logger?.error(`${TAG} Persona generation failed after ${elapsedMs}ms: ${err instanceof Error ? err.stack ?? err.message : String(err)}`);
|
||||
return false;
|
||||
}
|
||||
|
||||
// 9. Read LLM-written persona.md and apply post-processing
|
||||
let personaText: string;
|
||||
try {
|
||||
personaText = await fs.readFile(personaPath, "utf-8");
|
||||
} catch {
|
||||
// LLM failed to write persona.md — treat as failure
|
||||
this.logger?.error(`${TAG} LLM did not write persona.md — file not found after runner completed`);
|
||||
return false;
|
||||
}
|
||||
|
||||
// 10. Strip any navigation the LLM might have added + sanitize for safe injection
|
||||
personaText = escapeXmlTags(stripSceneNavigation(personaText).trim());
|
||||
|
||||
if (!personaText) {
|
||||
this.logger?.error(`${TAG} LLM wrote empty persona.md — skipping`);
|
||||
return false;
|
||||
}
|
||||
|
||||
// 11. Append fresh scene navigation and write final content
|
||||
const nav = generateSceneNavigation(index);
|
||||
const finalContent = nav ? `${personaText}\n\n${nav}\n` : personaText;
|
||||
await fs.writeFile(personaPath, finalContent, "utf-8");
|
||||
|
||||
// 12. Update checkpoint
|
||||
await cpManager.markPersonaGenerated(cp.total_processed);
|
||||
|
||||
const elapsedMs = Date.now() - startMs;
|
||||
this.logger?.info(`${TAG} Persona written (${finalContent.length} chars) in ${elapsedMs}ms`);
|
||||
|
||||
// ── l3_persona_generation metric ──
|
||||
if (this.instanceId && this.logger) {
|
||||
report("l3_persona_generation", {
|
||||
triggerReason: triggerReason ?? "unknown",
|
||||
mode: existingPersona ? "incremental" : "initial",
|
||||
newPersonaContent: personaText,
|
||||
newPersonaLength: personaText.length,
|
||||
totalDurationMs: elapsedMs,
|
||||
success: true,
|
||||
error: null,
|
||||
});
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
/**
|
||||
* PersonaTrigger: determines whether to trigger persona generation.
|
||||
* Implements the 5 trigger conditions from the legacy system.
|
||||
*/
|
||||
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { CheckpointManager } from "../utils/checkpoint.js";
|
||||
import { stripSceneNavigation } from "../scene/scene-navigation.js";
|
||||
|
||||
const TAG = "[memory-tdai] [trigger]";
|
||||
|
||||
interface TriggerLogger {
|
||||
debug?: (message: string) => void;
|
||||
info: (message: string) => void;
|
||||
warn: (message: string) => void;
|
||||
error: (message: string) => void;
|
||||
}
|
||||
|
||||
export interface TriggerResult {
|
||||
should: boolean;
|
||||
reason: string;
|
||||
}
|
||||
|
||||
export class PersonaTrigger {
|
||||
private dataDir: string;
|
||||
private interval: number;
|
||||
private logger: TriggerLogger | undefined;
|
||||
|
||||
constructor(opts: { dataDir: string; interval: number; logger?: TriggerLogger }) {
|
||||
this.dataDir = opts.dataDir;
|
||||
this.interval = opts.interval;
|
||||
this.logger = opts.logger;
|
||||
}
|
||||
|
||||
async shouldGenerate(): Promise<TriggerResult> {
|
||||
const cpManager = new CheckpointManager(this.dataDir);
|
||||
const cp = await cpManager.read();
|
||||
this.logger?.debug?.(`${TAG} Evaluating: total_processed=${cp.total_processed}, last_persona_at=${cp.last_persona_at}, memories_since=${cp.memories_since_last_persona}, scenes=${cp.scenes_processed}`);
|
||||
|
||||
// Priority 1: Agent explicitly requested persona update
|
||||
if (cp.request_persona_update) {
|
||||
const result: TriggerResult = {
|
||||
should: true,
|
||||
reason: `主动请求: ${cp.persona_update_reason || "Agent 请求更新"}`,
|
||||
};
|
||||
this.logger?.debug?.(`${TAG} Trigger P1 (explicit request): ${result.reason}`);
|
||||
return result;
|
||||
}
|
||||
|
||||
// Priority 2: Cold start — first extraction done, no persona yet, has scene files
|
||||
if (
|
||||
cp.scenes_processed > 0 &&
|
||||
cp.last_persona_at === 0 &&
|
||||
(await this.hasSceneFiles())
|
||||
) {
|
||||
const result: TriggerResult = { should: true, reason: "首次冷启动：首次提取完成且有场景文件" };
|
||||
this.logger?.debug?.(`${TAG} Trigger P2 (cold start): scenes_processed=${cp.scenes_processed}, total_processed=${cp.total_processed}`);
|
||||
return result;
|
||||
}
|
||||
|
||||
// Priority 2.5: Recovery — persona was generated before but persona.md body
|
||||
// is now empty (corrupted/missing). Regenerate to restore.
|
||||
if (
|
||||
cp.last_persona_at > 0 &&
|
||||
(await this.hasSceneFiles()) &&
|
||||
!(await this.hasPersonaBody())
|
||||
) {
|
||||
const result: TriggerResult = { should: true, reason: "恢复：persona.md 正文丢失或为空，需要重新生成" };
|
||||
this.logger?.debug?.(`${TAG} Trigger P2.5 (recovery): last_persona_at=${cp.last_persona_at}, persona body missing`);
|
||||
return result;
|
||||
}
|
||||
|
||||
// Priority 3: First scene block extraction
|
||||
if (cp.scenes_processed === 1 && cp.memories_since_last_persona > 0) {
|
||||
const result: TriggerResult = { should: true, reason: "首次 Scene Block 提取完成" };
|
||||
this.logger?.debug?.(`${TAG} Trigger P3 (first scene): scenes_processed=${cp.scenes_processed}`);
|
||||
return result;
|
||||
}
|
||||
|
||||
// Priority 4: Reached threshold
|
||||
if (cp.memories_since_last_persona >= this.interval) {
|
||||
const result: TriggerResult = {
|
||||
should: true,
|
||||
reason: `达到阈值: ${cp.memories_since_last_persona} >= ${this.interval}`,
|
||||
};
|
||||
this.logger?.debug?.(`${TAG} Trigger P4 (threshold): ${result.reason}`);
|
||||
return result;
|
||||
}
|
||||
|
||||
this.logger?.debug?.(`${TAG} No trigger conditions met`);
|
||||
return { should: false, reason: "" };
|
||||
}
|
||||
|
||||
private async hasSceneFiles(): Promise<boolean> {
|
||||
const blocksDir = path.join(this.dataDir, "scene_blocks");
|
||||
try {
|
||||
const files = await fs.readdir(blocksDir);
|
||||
const hasFiles = files.some((f) => f.endsWith(".md"));
|
||||
return hasFiles;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check whether persona.md has a non-empty body (excluding scene navigation).
|
||||
* Returns false if the file doesn't exist, is empty, or only contains
|
||||
* scene navigation (no actual persona content).
|
||||
*/
|
||||
private async hasPersonaBody(): Promise<boolean> {
|
||||
const personaPath = path.join(this.dataDir, "persona.md");
|
||||
try {
|
||||
const raw = await fs.readFile(personaPath, "utf-8");
|
||||
const body = stripSceneNavigation(raw).trim();
|
||||
return body.length > 0;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
/**
|
||||
* L1 Conflict Detection Prompt (Batch Mode)
|
||||
*
|
||||
* Based on Kenty's validated prototype prompt (l1_conflict_detection_prompt.md).
|
||||
* Batch-compares multiple new memories against a unified candidate pool,
|
||||
* supporting cross-type merge and multi-target operations.
|
||||
*/
|
||||
|
||||
import type { MemoryRecord, ExtractedMemory } from "../record/l1-writer.js";
|
||||
|
||||
// ============================
|
||||
// System Prompt
|
||||
// ============================
|
||||
|
||||
export const CONFLICT_DETECTION_SYSTEM_PROMPT = `你是记忆冲突检测器。批量比较多条【新记忆】与【统一候选记忆池】中的已有记忆，逐条决定如何处理。
|
||||
|
||||
## 核心规则
|
||||
|
||||
- **跨 type 合并**：不同 type（persona / episodic / instruction）的记忆如果语义上描述同一事实/事件，**可以合并**。
|
||||
- **多对多合并**：一条新记忆可以同时替换/合并候选池中的**多条**已有记忆（通过 target_ids 数组指定）。
|
||||
- 合并后你必须判断新记忆的最佳 type（merged_type）。
|
||||
|
||||
## 判断逻辑
|
||||
|
||||
1. **分辨记忆性质**：
|
||||
- **状态类**（persona/instruction）：偏好、特质、长期设定、相对稳定的事实、行为规则
|
||||
- **事件类**（episodic）：一次性经历、带时间点的客观记录，建议合并同一件事的前因后果
|
||||
|
||||
2. **判断是否同一事实/事件**：主体相同、主题一致、时间接近、scene_name 相似
|
||||
|
||||
3. **选择动作**：
|
||||
- "store"：视为新信息，新增当前记忆。
|
||||
- "skip"：已有记忆更好，新记忆无增量或更模糊，忽略当前记忆。
|
||||
- "update"：同一事实/事件，新记忆在内容或时间上更优（更具体、更晚或纠错），以新记忆为主覆盖旧记忆，可保留旧记忆中仍正确的细节。
|
||||
- "merge"：同一事实或同一演化过程，多条记忆信息互补且不矛盾，合并成一条更完整记忆，信息尽量不冗余。
|
||||
|
||||
4. **策略倾向**：
|
||||
- 状态类：多条描述同一偏好/特质 → 倾向 merge；无增量 → skip；明确更新 → update
|
||||
- 事件类：同一事件的前因后果、不同阶段 → 倾向 merge 为一条完整叙述；完全相同 → skip
|
||||
- 跨类型示例：一条 episodic "用户在 2018 年开始做播客" + 一条 persona "用户有播客制作经验" → 可 merge 为一条 persona 或 episodic（取决于信息侧重）
|
||||
|
||||
5. **timestamp 处理**：
|
||||
- merge / update 时，merged_timestamps 应包含**所有相关记忆的时间戳并集**（去重排序）
|
||||
- 这样可以保留事件发生的完整时间线
|
||||
|
||||
## 输出格式
|
||||
|
||||
严格输出 JSON 数组，每个元素对应一条新记忆的决策。不输出任何其他内容：
|
||||
|
||||
[
|
||||
{
|
||||
"record_id": "新记忆的 record_id",
|
||||
"action": "store|update|skip|merge",
|
||||
"target_ids": ["要删除的候选记忆 record_id 1", "record_id 2"],
|
||||
"merged_content": "合并/更新后的记忆内容（merge/update 时必填）",
|
||||
"merged_type": "合并后的最佳 type：persona|episodic|instruction（merge/update 时必填）",
|
||||
"merged_priority": 85,
|
||||
"merged_timestamps": ["合并后的时间戳数组，包含所有新旧记忆时间戳的并集（merge/update 时必填）"]
|
||||
}
|
||||
]
|
||||
|
||||
字段说明：
|
||||
- target_ids：要删除替换的旧记忆 ID **数组**（可以 1 条或多条）。store/skip 时省略或为空。
|
||||
- merged_content：merge/update 时的最终记忆文本。store/skip 时省略。
|
||||
- merged_type：merge/update 后记忆应归属的 type。根据合并后内容本质判断。
|
||||
- merged_priority：merge/update 后的新优先级（0-100 整数，merge/update 时必填）。合并后信息更完整、更确定，通常应**酌情提升** priority（例如两条 priority 70 的记忆合并后可提升到 80）。参考标准：80-100（核心特质/重要事件），60-79（一般偏好/普通活动），<60（次要信息）。
|
||||
- merged_timestamps：合并后的时间戳数组。收集新记忆 + 所有被合并旧记忆的时间戳，去重排序。`;
|
||||
|
||||
// ============================
|
||||
// Prompt Builder
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Candidate search result for a single new memory.
|
||||
*/
|
||||
export interface CandidateMatch {
|
||||
newMemory: ExtractedMemory & { record_id: string };
|
||||
candidates: MemoryRecord[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Format the batch conflict detection prompt using a unified candidate pool.
|
||||
*
|
||||
* Format (aligned with prototype):
|
||||
* 1. Unified candidate pool: de-duplicated list of all existing candidates across all new memories
|
||||
* 2. Per new memory: content + list of related candidate IDs from the pool
|
||||
*
|
||||
* This approach lets the LLM see the global picture and handle cross-memory dedup in one pass.
|
||||
*
|
||||
* @param matches - Array of new memories with their candidate matches
|
||||
*/
|
||||
export function formatBatchConflictPrompt(matches: CandidateMatch[]): string {
|
||||
// Step 1: Build unified candidate pool (de-duplicate across all new memories)
|
||||
const unifiedPool = new Map<string, MemoryRecord>();
|
||||
const perMemoryCandidateIds = new Map<string, string[]>();
|
||||
|
||||
for (const m of matches) {
|
||||
const candidateIds: string[] = [];
|
||||
for (const c of m.candidates) {
|
||||
if (!unifiedPool.has(c.id)) {
|
||||
unifiedPool.set(c.id, c);
|
||||
}
|
||||
candidateIds.push(c.id);
|
||||
}
|
||||
perMemoryCandidateIds.set(m.newMemory.record_id, candidateIds);
|
||||
}
|
||||
|
||||
// Step 2: Format unified pool as JSON
|
||||
const poolList = Array.from(unifiedPool.values()).map((c) => ({
|
||||
record_id: c.id,
|
||||
content: c.content,
|
||||
type: c.type,
|
||||
priority: c.priority,
|
||||
scene_name: c.scene_name,
|
||||
timestamps: c.timestamps,
|
||||
}));
|
||||
|
||||
let poolSection: string;
|
||||
if (poolList.length === 0) {
|
||||
poolSection = "## 统一候选记忆池\n\n（空，没有已有记忆，所有新记忆直接 store）";
|
||||
} else {
|
||||
const poolStr = JSON.stringify(poolList, null, 2);
|
||||
poolSection = `## 统一候选记忆池（共 ${poolList.length} 条已有记忆）\n\n${poolStr}`;
|
||||
}
|
||||
|
||||
// Step 3: Format each new memory with its related candidate IDs
|
||||
const memoryParts = matches.map((m, idx) => {
|
||||
const relatedIds = perMemoryCandidateIds.get(m.newMemory.record_id) ?? [];
|
||||
const relatedNote =
|
||||
relatedIds.length > 0
|
||||
? JSON.stringify(relatedIds)
|
||||
: "[]（无相似候选，直接 store）";
|
||||
|
||||
const memStr = JSON.stringify(
|
||||
{
|
||||
record_id: m.newMemory.record_id,
|
||||
content: m.newMemory.content,
|
||||
type: m.newMemory.type,
|
||||
priority: m.newMemory.priority,
|
||||
scene_name: m.newMemory.scene_name,
|
||||
},
|
||||
null,
|
||||
2,
|
||||
);
|
||||
|
||||
return `### 第 ${idx + 1} 条新记忆 (record_id: ${m.newMemory.record_id})\n${memStr}\n\n【关联候选 ID】${relatedNote}`;
|
||||
});
|
||||
|
||||
const newMemoriesText = memoryParts.join(
|
||||
"\n\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n\n",
|
||||
);
|
||||
|
||||
// Step 4: Assemble final prompt
|
||||
return `${poolSection}
|
||||
|
||||
${"═".repeat(50)}
|
||||
|
||||
## 待判断的新记忆（共 ${matches.length} 条）
|
||||
|
||||
${newMemoriesText}
|
||||
|
||||
请逐条判断并输出决策 JSON 数组。当某条新记忆的候选列表为空时，该条直接输出 action=store。`;
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
/**
|
||||
* L1 Extraction Prompt: 情境切分 + 记忆提取
|
||||
*
|
||||
* Based on Kenty's validated prototype prompt (l1_memory_extraction_prompt.md).
|
||||
* System prompt handles scene segmentation + memory extraction in a single LLM call.
|
||||
* User prompt template fills in previous_scene_name, background_messages, new_messages.
|
||||
*/
|
||||
|
||||
import type { ConversationMessage } from "../conversation/l0-recorder.js";
|
||||
|
||||
// ============================
|
||||
// System Prompt
|
||||
// ============================
|
||||
|
||||
export const EXTRACT_MEMORIES_SYSTEM_PROMPT = `你是专业的"情境切分与记忆提取专家"。
|
||||
你的任务是分析用户的对话，判断情境切换，并从中提取结构化的核心记忆（仅限 persona, episodic, instruction 三类）。
|
||||
|
||||
### 任务一：情境切分（Scene Segmentation）
|
||||
分析【待提取的新消息】，结合【上一个情境】，判断并输出当前对话的情境。
|
||||
- 继承：无明显切换，沿用上一个情境。
|
||||
- 切换条件：用户发出明确指令（如"换话题"）、意图转变、或提出独立新目标。
|
||||
- 一段对话可能只有一个情境，也可能有多个情境（话题多次切换时）。
|
||||
- 命名规则："我（AI）在和xxx（用户身份）做xxx（目标活动）"（中文，30-50字，单句，全局唯一）。
|
||||
|
||||
---
|
||||
|
||||
### 任务二：核心记忆提取（Memory Extraction）
|
||||
结合背景和当前情境，仅从【待提取的新消息】中提取核心信息。
|
||||
|
||||
【通用提取原则】
|
||||
1. 宁缺毋滥：过滤琐碎闲聊、临时性指令和一次性操作（如"这次、本单"）；剔除不可靠的边缘信息。
|
||||
2. 独立完整：记忆必须"跳出当前对话依然成立"，无上下文也能看懂。提取主体必须以"用户（姓名）"或"AI"为核心。
|
||||
3. 归纳合并：强关联或因果关系的多条消息，必须合并为一条完整记忆，不可碎片化。
|
||||
|
||||
【支持提取的三大类型】（必须严格遵守类型规则）
|
||||
|
||||
1. 个性化记忆 (type: "persona")
|
||||
- 定义：用户的稳定属性、偏好、技能、价值观、习惯（如住所、职业、饮食禁忌）。
|
||||
- 提取句式："用户（[姓名]）喜欢/是/擅长..."
|
||||
- 打分 (priority)：80-100（健康/禁忌/核心特质）；50-70（一般喜好/技能）；<50（模糊次要，可丢弃）。
|
||||
- 触发词：喜欢、习惯、经常、我这个人...
|
||||
|
||||
2. 客观事件记忆 (type: "episodic")
|
||||
- 定义：客观发生的动作、决定、计划或达成结果。绝不包含纯主观感受。
|
||||
- 提取句式："用户（[姓名]）在 [最好是精确绝对时间] 于 [地点] [做了某事（可以包含起因、经过、结果）]"。
|
||||
- 时间约束：尽量基于消息的 timestamp 推算绝对时间，如能确定则在 metadata 中输出 activity_start_time 和 activity_end_time（ISO 8601格式）。无法确定时可省略。
|
||||
- 打分 (priority)：80-100（重要事件/计划）；60-70（一般完整活动）；<60（琐碎事项，直接丢弃）。
|
||||
|
||||
3. 全局指令记忆 (type: "instruction")
|
||||
- 定义：用户对 AI 提出的长期行为规则、格式偏好、语气控制。
|
||||
- 提取句式："用户要求/希望 AI 以后回答时..."
|
||||
- 触发词：以后都、从现在开始、记住、必须。
|
||||
- 打分 (priority)：-1（极其严格的全局死命令）；90-100（核心行为规则）；70-80（重要要求）；<70（临时要求，直接丢弃）。
|
||||
|
||||
---
|
||||
|
||||
### 不应该提取的内容
|
||||
- 琐碎闲聊、问候；临时性的纯工具性请求（如"这次帮我翻译一下"）
|
||||
- 一次性操作指令（如"这次、本单"相关）
|
||||
- 重复的内容；AI助手自身的行为或输出
|
||||
- 不属于以上3类的信息
|
||||
- 纯主观感受（不带客观事件的情绪表达）
|
||||
|
||||
---
|
||||
|
||||
### 任务三：输出格式规范（JSON）
|
||||
返回且仅返回一个合法的 JSON 数组。数组的每一项是一个情境，包含该情境的消息范围和抽取到的记忆：
|
||||
|
||||
[
|
||||
{
|
||||
"scene_name": "当前生成或继承的情境名称",
|
||||
"message_ids": ["属于该情境的消息ID列表"],
|
||||
"memories": [
|
||||
{
|
||||
"content": "完整、独立的记忆陈述（按对应类型的句式要求）",
|
||||
"type": "persona|episodic|instruction",
|
||||
"priority": 80,
|
||||
"source_message_ids": ["消息ID_1", "消息ID_2"],
|
||||
"metadata": {}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
|
||||
metadata 字段说明：
|
||||
- episodic 类型：如能确定活动时间，填入 {"activity_start_time": "ISO8601", "activity_end_time": "ISO8601"}
|
||||
- 其他类型或无法确定时间：输出空对象 {}
|
||||
|
||||
如果整段对话无有意义的记忆，也要输出情境分割结果，memories 为空数组：
|
||||
[
|
||||
{
|
||||
"scene_name": "情境名称",
|
||||
"message_ids": ["id1", "id2"],
|
||||
"memories": []
|
||||
}
|
||||
]
|
||||
|
||||
请严格按上述 JSON 数组格式输出，不要输出任何额外的 Markdown 代码块修饰符（如 \`\`\`json）或解释文本。`;
|
||||
|
||||
// ============================
|
||||
// Prompt Builder
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Format the user prompt for L1 extraction.
|
||||
*
|
||||
* @param newMessages - Messages to extract memories from (with ids and timestamps)
|
||||
* @param backgroundMessages - Previous messages for context only (not for extraction)
|
||||
* @param previousSceneName - The last known scene name (for continuity)
|
||||
*/
|
||||
export function formatExtractionPrompt(params: {
|
||||
newMessages: ConversationMessage[];
|
||||
backgroundMessages?: ConversationMessage[];
|
||||
previousSceneName?: string;
|
||||
}): string {
|
||||
const { newMessages, backgroundMessages = [], previousSceneName = "无" } = params;
|
||||
|
||||
const bgText = backgroundMessages.length > 0
|
||||
? backgroundMessages
|
||||
.map((m) => `[${m.id}] [${m.role}] [${new Date(m.timestamp).toISOString()}]: ${m.content}`)
|
||||
.join("\n\n")
|
||||
: "无";
|
||||
|
||||
const newText = newMessages
|
||||
.map((m) => `[${m.id}] [${m.role}] [${new Date(m.timestamp).toISOString()}]: ${m.content}`)
|
||||
.join("\n\n");
|
||||
|
||||
return `【上一个情境】：${previousSceneName}
|
||||
|
||||
【背景对话】（仅供理解上下文推断关系/时间，严禁从中提取记忆）：
|
||||
${bgText}
|
||||
|
||||
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||
|
||||
【待提取的新消息】（务必结合 timestamp 推算时间，只从这里提取记忆！）：
|
||||
${newText}`;
|
||||
}
|
||||
@@ -0,0 +1,172 @@
|
||||
/**
|
||||
* Persona Generation Prompt — instructs LLM to generate/update user persona
|
||||
* using the four-layer deep scan model.
|
||||
*
|
||||
* v2: Updated prompt with anti-hallucination guardrails, richer output
|
||||
* template with per-chapter writing guidance, and streamlined iteration guide.
|
||||
*/
|
||||
|
||||
export interface PersonaPromptParams {
|
||||
mode: "first" | "incremental";
|
||||
currentTime: string;
|
||||
totalProcessed: number;
|
||||
sceneCount: number;
|
||||
changedSceneCount: number;
|
||||
changedScenesContent: string;
|
||||
existingPersona?: string;
|
||||
triggerInfo?: string;
|
||||
/** @deprecated Kept for call-site compatibility; no longer used in prompt. */
|
||||
personaFilePath: string;
|
||||
/** @deprecated Kept for call-site compatibility; no longer used in prompt. */
|
||||
checkpointPath: string;
|
||||
}
|
||||
|
||||
export function buildPersonaPrompt(params: PersonaPromptParams): string {
|
||||
const {
|
||||
mode,
|
||||
currentTime,
|
||||
totalProcessed,
|
||||
sceneCount,
|
||||
changedSceneCount,
|
||||
changedScenesContent,
|
||||
existingPersona,
|
||||
triggerInfo,
|
||||
} = params;
|
||||
|
||||
const modeLabel = mode === "first" ? "🆕 首次生成" : "🔄 迭代更新";
|
||||
|
||||
const triggerSection = triggerInfo
|
||||
? `\n### 触发信息\n${triggerInfo}\n`
|
||||
: "";
|
||||
|
||||
// Existing persona is now placed AFTER scene blocks (closer to the
|
||||
// processing instructions) so the LLM sees fresh evidence first.
|
||||
const existingPersonaSection = existingPersona
|
||||
? `\n## 📄 当前 Persona（工程已预加载）\n\n` +
|
||||
`*以下是现有 persona.md 的完整内容（${existingPersona.length} 字符），基于此更新后请控制在2000字内：*\n\n` +
|
||||
`\`\`\`markdown\n${existingPersona}\n\`\`\`\n\n---\n`
|
||||
: "";
|
||||
|
||||
const iterationGuide = mode === "incremental"
|
||||
? `\n## 🔄 迭代决策指南\n\n` +
|
||||
`面对变化场景，自主判断处理方式：强化（佐证已有洞察）/ 补充（新维度）/ 修正（矛盾）/ 重构（结构调整）/ 不改（无有用新增内容）。\n`
|
||||
: "";
|
||||
|
||||
return `# 🧬 Persona Architect - Incremental Evolution Protocol
|
||||
|
||||
**⏰ 更新时间**: ${currentTime}
|
||||
**模式**: ${modeLabel}
|
||||
|
||||
请你结合已有的 persona.md 和新增/变化的 block 信息深度分析，然后使用文件工具将结果写入 \`persona.md\` 文件。
|
||||
|
||||
## ⛔ 文件操作约束（必须严格遵守）
|
||||
|
||||
1. **必须使用文件工具将最终 persona 内容写入 \`persona.md\`**。当前工作目录已设为数据目录，直接使用文件名 \`persona.md\`。
|
||||
- **首次生成 / 大幅重写**：使用 \`write_to_file('persona.md', 完整内容)\` 整体写入
|
||||
- **增量更新（局部修改）**：使用 \`replace_in_file('persona.md', 旧内容片段, 新内容片段)\` 精确替换，可节省开销
|
||||
2. **只能操作 \`persona.md\` 这一个文件**，禁止读取或写入任何其他文件（包括 scene_blocks/、.metadata/ 等）。
|
||||
3. **写入的内容必须只包含最终的 persona 文档**，不要包含你的思考过程、分析步骤或任何非 persona 内容。
|
||||
4. **无需 read_file**：当前 persona.md 的完整内容已在下方提供，直接基于它进行更新即可。
|
||||
|
||||
### 🚫 严格禁止
|
||||
- **禁止过长**：persona.md 内容总长度不要超过 2000 字符，及时做总结和删除不重要的信息。
|
||||
- **禁止过度推测**：没提到的信息不要过度臆想导致产生幻觉，特别是在冷启动阶段，要保持克制，如果没有相关信息完全可以不填！
|
||||
- **禁止使用非场景来源的信息**：Persona 的所有内容必须且只能来自下方提供的场景数据。不要从 workspace 目录结构、文件路径、系统信息等技术元数据中提取任何关于用户的个人信息。
|
||||
- **禁止操作 persona.md 以外的任何文件**。
|
||||
|
||||
---
|
||||
${triggerSection}
|
||||
## 📊 统计
|
||||
- **总记忆数**: ${totalProcessed} 条
|
||||
- **场景总数**: ${sceneCount} 个
|
||||
- **变化场景**: ${changedSceneCount} 个（自上次更新后）
|
||||
|
||||
---
|
||||
${changedScenesContent}
|
||||
|
||||
|
||||
${existingPersonaSection}
|
||||
## ⚙️ 核心运作逻辑 (The Core Logic)
|
||||
|
||||
🧠 核心思维引擎：连接与综合 (Connect & Synthesize)
|
||||
请遵循 "叙事连贯性" 原则处理信息。禁止简单的罗列（No Bullet-point Spamming）。
|
||||
|
||||
1. 寻找"贯穿线" (The Connecting Thread)
|
||||
不要孤立地看信息。要寻找不同领域行为背后的共同逻辑。
|
||||
** 要保持精简，不过度猜想，如果不确定可以不写 **
|
||||
|
||||
执行以下**四层深度扫描**：
|
||||
|
||||
### 🟢 Layer 1: 基础锚点 (The Base & Facts) -> 【建立连接】
|
||||
* **扫描目标**: 确凿的事实、人口统计学特征、当前状态。
|
||||
* **实用价值**: 为 Agent 提供**破冰话题**和**上下文感知**。
|
||||
|
||||
### 🔵 Layer 2: 兴趣图谱 (The Interest Graph) -> 【提供谈资】
|
||||
* **扫描目标**: 用户投入时间、金钱或注意力的事物。
|
||||
* **提取原则**: **区分活跃度**（活跃爱好 / 被动消费 / 休眠兴趣）。
|
||||
* **实用价值**: 让 Agent 能够进行**高质量的闲聊 (Chit-chat)** 和 **生活推荐**。
|
||||
|
||||
### 🟡 Layer 3: 交互协议 (The Interface) -> 【消除摩擦】
|
||||
* **扫描目标**: 用户的沟通习惯、雷区、工作流偏好。
|
||||
* **实用价值**: 指导 Agent **如何说话、如何交付结果**，避免踩雷。
|
||||
|
||||
### 🔴 Layer 4: 认知内核 (The Core) -> 【深度共鸣】
|
||||
* **扫描目标**: 决策逻辑、矛盾点、终极驱动力。
|
||||
* **实用价值**: 让 Agent 成为**能够替用户做决策**的"副驾驶"。
|
||||
${iterationGuide}
|
||||
---
|
||||
|
||||
## 📝 输出模板 (The Persona Template)
|
||||
|
||||
请参考以下格式，使用 \`write_to_file('persona.md', ...)\` 写入最终内容。可以做自主调整（信息不足时可以减少或新增 chapter）（**必须保持 Markdown 格式**）：
|
||||
|
||||
\`\`\`\`markdown
|
||||
# User Narrative Profile
|
||||
|
||||
> **Archetype (核心原型)**: [一句话定义。例如：一位在现实重力下挣扎，但试图通过技术构建理想国的"务实理想主义者"。]
|
||||
|
||||
> **基本信息**
|
||||
（用户的基本信息，如年龄、性别、职业等，更新时若有冲突则覆盖，不冲突尽量叠加）
|
||||
-
|
||||
-
|
||||
|
||||
> **长期偏好**
|
||||
（你观察到的用户最稳定且可复用的偏好）
|
||||
-
|
||||
-
|
||||
|
||||
## 📖 Chapter 1: Context & Current State (全景语境)
|
||||
*(将基础事实与当前状态融合，写成一段连贯的背景介绍)*
|
||||
|
||||
**[这里写连贯描述，区别较大的时候可以分点阐述]**
|
||||
|
||||
## 🎨 Chapter 2: The Texture of Life (生活的肌理)
|
||||
*(将兴趣、消费、生活习惯串联起来，展示生活品味)*
|
||||
|
||||
**[这里写连贯的描述，重点在于"兴趣/偏好"和"品味"的统一性，区别较大的时候可以分点阐述]**
|
||||
|
||||
## 🤖 Chapter 3: Interaction & Cognitive Protocol (交互与认知协议)
|
||||
*(这是 Main Agent 的行动指南。为了实用，这里保持半结构化，但要解释"为什么")*
|
||||
|
||||
### 3.1 沟通策略 (How to Speak)
|
||||
### 3.2 决策逻辑 (How to Think)
|
||||
|
||||
## 🧩 Chapter 4: Deep Insights & Evolution (深层洞察与演变)
|
||||
*(人类学观察笔记)*
|
||||
|
||||
* **矛盾统一性**: [描述用户身上看似冲突但实则合理的特质]。
|
||||
* **演变轨迹**: [可加上时间，分为多点，描述用户最近发生的变化]。
|
||||
* **涌现特征**: 提炼 3-7 个最核心的特质标签，每个标签单独一行并附上简短注释（10-15字）
|
||||
- \`TagName\` - 简短注释说明
|
||||
\`\`\`\`
|
||||
|
||||
---
|
||||
|
||||
### ⚠️ 成功标准
|
||||
- ✅ **必须使用 \`write_to_file\` 或 \`replace_in_file\` 工具写入最终结果到 \`persona.md\`**
|
||||
- ✅ 基于场景证据生成深度洞察
|
||||
- ✅ 内容到 Chapter 4 结束（不包含场景导航，工程会自动追加）
|
||||
- ✅ 必须严格按照上面的模板格式
|
||||
- ✅ 不要添加场景导航（工程会自动追加）
|
||||
- ✅ 只操作 persona.md，不要操作其他文件`;
|
||||
}
|
||||
@@ -0,0 +1,228 @@
|
||||
/**
|
||||
* Scene Extraction Prompt — instructs LLM to consolidate memories into scene blocks
|
||||
* using file tools (read_file, write_to_file, replace_in_file).
|
||||
*
|
||||
* Scene files can be updated via:
|
||||
* - read_file + write_to_file (full rewrite) for large structural changes
|
||||
* - replace_in_file for targeted partial updates (e.g. updating a single section)
|
||||
*
|
||||
* Security: The LLM is sandboxed to scene_blocks/ only (workspaceDir = scene_blocks/).
|
||||
* It has NO visibility into checkpoint, scene_index, persona.md, or any other system file.
|
||||
* File deletion is achieved via "soft-delete" — writing an empty string to the file
|
||||
* — and the SceneExtractor subsequently removes empty files with fs.unlink.
|
||||
*
|
||||
* Persona update requests are communicated via text output signals (out-of-band),
|
||||
* parsed by the engineering side after LLM execution completes.
|
||||
*/
|
||||
|
||||
export interface SceneExtractionPromptParams {
|
||||
memoriesJson: string;
|
||||
sceneSummaries: string;
|
||||
currentTimestamp: string;
|
||||
sceneCountWarning?: string;
|
||||
/** List of existing scene filenames (relative, e.g. ["work.md", "hobby.md"]) */
|
||||
existingSceneFiles?: string[];
|
||||
}
|
||||
|
||||
export function buildSceneExtractionPrompt(params: SceneExtractionPromptParams): string {
|
||||
const {
|
||||
memoriesJson,
|
||||
sceneSummaries,
|
||||
currentTimestamp,
|
||||
sceneCountWarning,
|
||||
existingSceneFiles,
|
||||
} = params;
|
||||
|
||||
const warningSection = sceneCountWarning
|
||||
? `\n⚠️ **场景数量警告**: ${sceneCountWarning}\n`
|
||||
: "";
|
||||
|
||||
return `# System Prompt: Memory Consolidation Architect
|
||||
|
||||
## 角色定义 (Role Definition)
|
||||
你是记忆整合架构师。你的目标是为用户构建一个"数字第二大脑"。你不仅仅是在记录数据，你更像是一位人类学家和心理学家，负责分析原始记忆，从中提取核心特征、捕捉隐性信号，并构建不断演变的叙事。
|
||||
|
||||
|
||||
## 架构模型
|
||||
|
||||
### Layer 1 (Input): Raw Memories
|
||||
- **来源**：API 分批召回（每批 20 条）
|
||||
- **状态**：碎片化、无序
|
||||
|
||||
### Layer 2 (Processing): Scene Diaries
|
||||
- **形态**：**不是清单，是连贯的叙事文档**
|
||||
- **逻辑**：将 L1 碎片融合进特定场景文件（强制限制在15个以内）
|
||||
- **动作**：Create（创建）、Integrate（整合）、Rewrite（重写）
|
||||
- **禁止**：简单追加列表
|
||||
|
||||
你主要负责L1到L2的生成任务
|
||||
${warningSection}
|
||||
## 输入环境 (Input Context)
|
||||
你将接收三个输入：
|
||||
1. 新增记忆 (New Memory): 一段原始的、非结构化的新近回忆信息。
|
||||
2. 现有 Block 映射表 (Existing Blocks Map): 包含当前所有记忆块（Markdown 文件）的文件名和摘要的列表。
|
||||
3. 当前时间 (Current Time): 用于生成元数据的具体时间戳。
|
||||
|
||||
|
||||
### 1️⃣ New Memories List
|
||||
${memoriesJson}
|
||||
|
||||
### 2️⃣ Existing Scene Blocks Summary
|
||||
${sceneSummaries}
|
||||
|
||||
### 3️⃣ Current Timestamp
|
||||
${currentTimestamp}
|
||||
|
||||
${existingSceneFiles && existingSceneFiles.length > 0
|
||||
? `### 📁 已有场景文件清单（仅以下文件可 read_file）
|
||||
${existingSceneFiles.map((f) => `- \`${f}\``).join("\n")}
|
||||
`
|
||||
: `### 📁 已有场景文件清单
|
||||
（当前无已有场景文件）
|
||||
`}
|
||||
## ⛔ 文件操作约束（必须严格遵守）
|
||||
1. **所有文件操作使用相对文件名**（如 \`技术研究-Rust学习.md\`），当前工作目录已设为场景文件目录
|
||||
2. **read_file 只能读取上面"已有场景文件清单"中列出的文件**，禁止猜测或编造不在清单中的文件名
|
||||
3. **创建新场景文件时**，直接使用文件名，如 \`新场景名.md\`
|
||||
4. **场景文件支持 replace_in_file**。对于局部更新（如只更新某个章节或 META 字段），可以使用 \`replace_in_file\` 进行精确替换。对于大范围重写或结构性变更，建议使用 \`read_file\` + \`write_to_file\` 整体重写。
|
||||
5. **场景索引和系统配置由工程系统自动维护**，你只需专注于操作 \`.md\` 场景文件
|
||||
|
||||
## 工作流与逻辑 (Workflow & Logic)
|
||||
在生成输出之前，你必须执行以下"思维链"过程：
|
||||
|
||||
### ⚠️ 阶段 0：强制检查场景总数（必须先执行）
|
||||
|
||||
**在处理任何记忆之前，你必须：**
|
||||
|
||||
1. **统计当前场景总数**：检查 "Existing Scene Blocks Summary" 中的场景数量
|
||||
2. **遵守分级预警，上限为15个block**：
|
||||
- 红色预警（≥ 15）：**必须先合并**，将最相似的 2-4 个场景合并为 1 个，然后再处理新记忆
|
||||
- 橙色预警（= 15-1）：**只能 UPDATE 现有场景，不能 CREATE 新场景**
|
||||
- 黄色预警（接近15）：**优先 UPDATE 或主动 MERGE 相似场景**
|
||||
|
||||
**合并优先级**（当需要合并时，按以下顺序选择）：
|
||||
1. **主题高度重叠**：如"Python后端开发"和"Go后端开发" → 合并为"后端开发技术栈"
|
||||
2. **叙事弧线相同**：如"求职材料-JD匹配"和"职业发展-能力对齐" → 合并为"职业发展与求职"
|
||||
3. **热度最低的场景**：如果没有明显重叠，合并或删除 heat 最低的 2-3 个场景
|
||||
|
||||
### 阶段 1：分析与分类
|
||||
分析 新增记忆。它的核心领域是什么？（例如：编程风格、情绪状态、职业轨迹、人际关系）。
|
||||
提取事实事件链（触发 -> 行动 -> 结果）以及底层的心理状态。
|
||||
|
||||
### 阶段 2：检索与策略选择
|
||||
将新记忆与 现有 Block 映射表 进行比对。
|
||||
需要时使用 \`read_file\` 工具读取完整场景文件内容
|
||||
**只能读取上面"已有场景文件清单"中列出的文件，禁止猜测其他文件路径。**
|
||||
|
||||
**核心原则：默认策略是 UPDATE，不是 CREATE。** 当犹豫于 UPDATE 和 CREATE 之间时，选择 UPDATE。
|
||||
|
||||
策略选择（按优先级排序）：
|
||||
1. **UPDATE（更新）**【首选策略】: 如果存在相关的 Block（基于摘要或文件名的相似性），先 read 文件内的具体信息，再锁定该 Block 进行更新（write 或 replace）
|
||||
2. **MERGE（合并）**:
|
||||
- 合并的新 block 应该是生成概括性更强的场景，包含已有的多个相似场景
|
||||
- **强制合并**：当前 Block 总数 **≥ 上限**时，必须先将多个相似记忆合并，或者删除最旧或者最不重要的 block
|
||||
- **主动合并**：即使未达上限，如果两个 Block 属于同一叙事弧线，也应合并以增加深度
|
||||
3. **CREATE（新建）**【最后手段】:
|
||||
- **前提条件**：当前场景总数未达上限
|
||||
- **CREATE 前的强制验证**：必须先用 \`read_file\` 检查至少 2 个最相似的现有场景，确认新记忆确实无法融入后才能 CREATE。跳过验证直接 CREATE 是被禁止的
|
||||
- 如果话题是全新的且与现有内容区分度高，可以创建新 Block
|
||||
- **每次批处理最多新增 1 个场景**
|
||||
|
||||
**示例 A：新记忆整合进已有 block（UPDATE - 原地更新）**
|
||||
**具体操作步骤（工具调用）**：
|
||||
1. \`read_file('Python后端开发.md')\` → 获取已有内容 A
|
||||
2. 分析新记忆 + 已有内容 A → 整合生成新内容 B（\`heat = 旧heat + 1\`）
|
||||
3. \`write_to_file('Python后端开发.md', B)\` → **整体重写该场景文件**
|
||||
或 \`replace_in_file('Python后端开发.md', old_section, new_section)\` → **局部更新某部分**
|
||||
|
||||
合并多个 block 的逻辑：
|
||||
**具体操作步骤（工具调用）**：
|
||||
1. \`read_file('Python后端开发.md')\` → 获取内容 A
|
||||
2. \`read_file('Go后端开发.md')\` → 获取内容 B
|
||||
3. 整合 A + B + 新记忆 → 生成新内容 C（\`heat = heatA + heatB + 1\`）
|
||||
4. \`write_to_file('后端开发技术栈.md', C)\` → 创建新文件，写入合并后的完整内容
|
||||
5. \`write_to_file('Python后端开发.md', '')\` → **清空旧文件 A（标记删除）**
|
||||
6. \`write_to_file('Go后端开发.md', '')\` → **清空旧文件 B（标记删除）**
|
||||
|
||||
### 阶段 3：撰写与合成（核心任务）
|
||||
深度整合: 严禁简单的文本追加。你必须结合上下文（基于摘要或提供的原始内容）重写叙事，将新信息自然地融入其中。
|
||||
隐性推断: 寻找用户 没说出口 的信息。更新"隐性信号"部分。
|
||||
冲突检测: 如果新记忆与旧记忆相矛盾，将其记录在"演变轨迹"或"待确认/矛盾点"中。
|
||||
|
||||
### 撰写准则 (严格遵守)
|
||||
核心部分禁止列表: "用户核心特征"和"核心叙事"必须是连贯的段落，信息要连贯，可以分段。
|
||||
叙事弧线: "核心叙事"必须遵循故事结构（情境 -> 行动 -> 结果）。
|
||||
|
||||
### 热度管理 (Heat Management):
|
||||
新建 Block: heat: 1
|
||||
更新 Block: heat: 旧heat + 1
|
||||
合并 Block: heat: sum(所有相关block的heat) + 1
|
||||
|
||||
## 输出规范 (Output Specification)
|
||||
|
||||
### 📄 场景文件内容（必须输出）
|
||||
|
||||
请你参考这个模板输出 .md 文件的内容或基于已有md进行更新，每个md控制在1500字符内。不要把模板本身放在 Markdown 代码块中，只需直接输出要写入文件的原始文本。
|
||||
|
||||
\`\`\`markdown
|
||||
-----META-START-----
|
||||
created: {{EXISTING_CREATED_TIME_OR_CURRENT_TIME}}
|
||||
updated: {{CURRENT_TIME}}
|
||||
summary: [30-40 words concise summary for indexing]
|
||||
heat: [Integer]
|
||||
-----META-END-----
|
||||
|
||||
## 用户基础信息
|
||||
[可为空，如果没有可不写这节，可按照需求添加更多点，合并和更新方式尽量叠加，有冲突则覆盖]
|
||||
-姓名：
|
||||
-职业：
|
||||
-居住地：
|
||||
- ……
|
||||
|
||||
## 用户核心特征
|
||||
[这里不是列表！是一段连贯的描述。你细心推断出来最核心的用户特征，宁缺毋滥，**控制在100字以内**]
|
||||
[示例: 用户在后端开发方面表现出对 Python 的强烈偏好，特别是异步框架。近期（2026-02）开始关注 Rust 的所有权机制，这表明用户有向系统级编程转型的意图。]
|
||||
|
||||
## 用户偏好
|
||||
[这里可以是列表！**如果没有可以为不写这节**，记录用户明确的偏好信息（显性偏好），注意不要重复信息，不要流水账，偏好要可复用，更新时可以动态整合甚至重写]
|
||||
[示例：用户喜欢吃苹果]
|
||||
|
||||
## 隐性信号
|
||||
[这是给人类学家看的，记录那些"没明说但很重要"的事，和显性偏好不一样，一定是你推断出来的，需要深思熟虑后再生成，可以为空，宁缺毋滥。你可以随时更新/删除/修改这里的信息]
|
||||
|
||||
## 核心叙事
|
||||
[这里不是列表！是一段连贯的描述，**控制在400字以内**，注意不要重复信息，不要流水账，可以动态整合甚至重写]
|
||||
*(这里记录连贯的故事，必须包含 Trigger -> Action -> Result)*
|
||||
|
||||
[ 示例：本周用户主要集中在后端重构上。初期因为旧代码的耦合度高感到沮丧（**情绪点**），但他拒绝了"打补丁"的建议，坚持进行彻底解耦（**决策点**）。他在此过程中频繁查阅架构设计模式，表现出对"代码洁癖"的执着。]
|
||||
|
||||
|
||||
## 演变轨迹
|
||||
> [注意] 可以为空，仅记录【用户偏好/性格/重大观念】转变，不记录琐碎、日常更新。当发生冲突时，不要直接覆盖，要记录变化轨迹。
|
||||
- [2026-01-10]: 从 "反对加班" 转向 "接受弹性工作"，原因：创业压力（记忆ID: #987）
|
||||
|
||||
|
||||
## 待确认/矛盾点
|
||||
- [记录当前无法整合的矛盾信息，等待未来记忆澄清]
|
||||
|
||||
\`\`\`
|
||||
|
||||
|
||||
|
||||
#### 主动触发 Persona 更新（可选）
|
||||
|
||||
**触发条件**：重大价值观转变、跨场景突破性洞察。
|
||||
|
||||
**触发方式**：在你的 text output 中输出以下标记（不是文件操作）：
|
||||
|
||||
[PERSONA_UPDATE_REQUEST]
|
||||
reason: 具体原因描述
|
||||
[/PERSONA_UPDATE_REQUEST]
|
||||
|
||||
|
||||
**执行文件操作**（必须使用工具）：
|
||||
- 使用 \`read_file\` 读取需要更新的场景文件
|
||||
- 使用 \`write_to_file\` 创建新文件或**整体重写**已有场景文件
|
||||
- 使用 \`replace_in_file\` 对场景文件进行**局部更新**（如只更新某个章节）
|
||||
- **删除文件**：使用 \`write_to_file(filename, '')\` 将文件内容清空（系统会自动清理空文件，禁止使用其他删除方式）`;
|
||||
}
|
||||
@@ -0,0 +1,382 @@
|
||||
/**
|
||||
* L1 Memory Conflict Detection (Batch Mode): decides how to handle multiple new
|
||||
* memories against existing records in a single LLM call.
|
||||
*
|
||||
* v4: Removed JSONL-based Jaccard fallback. Candidate recall now relies exclusively
|
||||
* on vector search (primary) and FTS5 BM25 (degraded). If neither is available,
|
||||
* conflict detection is skipped entirely — all memories go straight to store.
|
||||
*
|
||||
* Two-phase approach:
|
||||
* 1. Candidate search per new memory — vector recall or FTS5 keyword recall (fast, no LLM)
|
||||
* 2. Batch LLM judgment on all new memories + their candidate pools (single call)
|
||||
*/
|
||||
|
||||
import type { ExtractedMemory, MemoryRecord, DedupDecision, MemoryType } from "./l1-writer.js";
|
||||
import { CONFLICT_DETECTION_SYSTEM_PROMPT, formatBatchConflictPrompt } from "../prompts/l1-dedup.js";
|
||||
import type { CandidateMatch } from "../prompts/l1-dedup.js";
|
||||
import { CleanContextRunner } from "../utils/clean-context-runner.js";
|
||||
import { sanitizeJsonForParse } from "../utils/sanitize.js";
|
||||
import type { VectorStore } from "../store/vector-store.js";
|
||||
import { buildFtsQuery } from "../store/vector-store.js";
|
||||
import type { EmbeddingService } from "../store/embedding.js";
|
||||
|
||||
interface Logger {
|
||||
debug?: (message: string) => void;
|
||||
info: (message: string) => void;
|
||||
warn: (message: string) => void;
|
||||
error: (message: string) => void;
|
||||
}
|
||||
|
||||
const TAG = "[memory-tdai][l1-dedup]";
|
||||
|
||||
// ============================
|
||||
// Core function (batch mode)
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Batch conflict detection: compare all new memories against existing records
|
||||
* in a single LLM call.
|
||||
*
|
||||
* Candidate recall strategy (3-tier degradation):
|
||||
* 1. Vector recall (vectorStore + embeddingService) — cosine similarity (best)
|
||||
* 2. FTS5 keyword recall (vectorStore with FTS available) — BM25 ranking (degraded)
|
||||
* 3. Skip conflict detection entirely — all memories go straight to "store"
|
||||
*
|
||||
* The old JSONL-based Jaccard fallback has been removed. If neither vector search
|
||||
* nor FTS is available, we skip dedup rather than paying the O(N) full-file-scan cost.
|
||||
*
|
||||
* @param memories - Newly extracted memories (with record_id)
|
||||
* @param config - OpenClaw config (for LLM access)
|
||||
* @param logger - Optional logger
|
||||
* @param model - Optional model override
|
||||
* @param vectorStore - Optional vector store for cosine similarity search
|
||||
* @param embeddingService - Optional embedding service for computing query vectors
|
||||
* @param conflictRecallTopK - Top-K candidates to recall per new memory (default: 5)
|
||||
* @returns Array of dedup decisions, one per new memory
|
||||
*/
|
||||
export async function batchDedup(params: {
|
||||
memories: Array<ExtractedMemory & { record_id: string }>;
|
||||
config: unknown;
|
||||
logger?: Logger;
|
||||
model?: string;
|
||||
/** Vector store for cosine similarity candidate recall */
|
||||
vectorStore?: VectorStore;
|
||||
/** Embedding service for computing query vectors */
|
||||
embeddingService?: EmbeddingService;
|
||||
/** Top-K candidates per new memory (default: 5) */
|
||||
conflictRecallTopK?: number;
|
||||
}): Promise<DedupDecision[]> {
|
||||
const { memories, config, logger, model, vectorStore, embeddingService } = params;
|
||||
const topK = params.conflictRecallTopK ?? 5;
|
||||
|
||||
if (memories.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const storeAll = () =>
|
||||
memories.map((m) => ({
|
||||
record_id: m.record_id,
|
||||
action: "store" as const,
|
||||
target_ids: [],
|
||||
}));
|
||||
|
||||
// Determine what recall capabilities are available
|
||||
const hasVectorData = vectorStore && vectorStore.count() > 0;
|
||||
const hasFts = vectorStore?.isFtsAvailable() ?? false;
|
||||
|
||||
// Fast path: no recall capability at all → skip dedup
|
||||
if (!hasVectorData && !hasFts) {
|
||||
logger?.debug?.(`${TAG} No vector data and no FTS available, skipping conflict detection for ${memories.length} memories`);
|
||||
return storeAll();
|
||||
}
|
||||
|
||||
// Phase 1: Find candidates
|
||||
//
|
||||
// Decision tree (after the fast-path guard above, vectorStore is guaranteed non-null):
|
||||
// hasVectorData + embeddingService → Tier 1 vector recall (FTS fallback on error)
|
||||
// otherwise hasFts → Tier 2 FTS keyword recall
|
||||
// otherwise → skip dedup (defensive; shouldn't reach here)
|
||||
let matches: CandidateMatch[];
|
||||
|
||||
if (hasVectorData && embeddingService) {
|
||||
// === Tier 1: Vector recall mode ===
|
||||
logger?.debug?.(`${TAG} Using vector recall mode (topK=${topK})`);
|
||||
try {
|
||||
matches = await findCandidatesByVector(memories, vectorStore!, embeddingService, topK, logger);
|
||||
} catch (err) {
|
||||
logger?.warn?.(
|
||||
`${TAG} Vector recall failed, falling back to FTS keyword: ${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
// Degrade to FTS keyword recall
|
||||
if (hasFts) {
|
||||
matches = findCandidatesByFts(memories, vectorStore!, logger);
|
||||
} else {
|
||||
logger?.debug?.(`${TAG} FTS not available either, skipping conflict detection`);
|
||||
return storeAll();
|
||||
}
|
||||
}
|
||||
} else if (hasFts) {
|
||||
// === Tier 2: FTS keyword recall ===
|
||||
logger?.debug?.(`${TAG} Using FTS keyword recall mode (no embedding service or no vector data)`);
|
||||
matches = findCandidatesByFts(memories, vectorStore!, logger);
|
||||
} else {
|
||||
// Shouldn't reach here given the fast-path check above, but be defensive
|
||||
logger?.debug?.(`${TAG} No usable recall path, skipping conflict detection`);
|
||||
return storeAll();
|
||||
}
|
||||
|
||||
// Check if any memory has candidates
|
||||
const hasAnyCandidates = matches.some((m) => m.candidates.length > 0);
|
||||
|
||||
if (!hasAnyCandidates) {
|
||||
logger?.debug?.(`${TAG} No similar records found for any memory, all will be stored`);
|
||||
return storeAll();
|
||||
}
|
||||
|
||||
// Phase 2: Batch LLM judgment
|
||||
return runLlmJudgment(matches, memories, config, logger, model);
|
||||
}
|
||||
|
||||
/**
|
||||
* Phase 2: Run batch LLM judgment on candidate matches.
|
||||
*/
|
||||
async function runLlmJudgment(
|
||||
matches: CandidateMatch[],
|
||||
memories: Array<ExtractedMemory & { record_id: string }>,
|
||||
config: unknown,
|
||||
logger: Logger | undefined,
|
||||
model: string | undefined,
|
||||
): Promise<DedupDecision[]> {
|
||||
logger?.debug?.(`${TAG} Running batch conflict detection for ${memories.length} memories`);
|
||||
|
||||
try {
|
||||
const runner = new CleanContextRunner({
|
||||
config,
|
||||
modelRef: model,
|
||||
enableTools: false,
|
||||
logger,
|
||||
});
|
||||
|
||||
const userPrompt = formatBatchConflictPrompt(matches);
|
||||
|
||||
const result = await runner.run({
|
||||
prompt: userPrompt,
|
||||
systemPrompt: CONFLICT_DETECTION_SYSTEM_PROMPT,
|
||||
taskId: "l1-conflict-detection",
|
||||
timeoutMs: 180_000,
|
||||
// maxTokens: 4000, remove maxTokens use model default or inherit from config
|
||||
});
|
||||
|
||||
const decisions = parseBatchResult(result, memories, logger);
|
||||
return decisions;
|
||||
} catch (err) {
|
||||
logger?.warn?.(
|
||||
`${TAG} Batch conflict detection failed, defaulting all to store: ${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
return memories.map((m) => ({
|
||||
record_id: m.record_id,
|
||||
action: "store" as const,
|
||||
target_ids: [],
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Candidate recall strategies
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Vector-based candidate recall (aligned with prototype):
|
||||
* batch-embed new memories → cosine search in VectorStore → exclude self-batch → return candidates.
|
||||
*/
|
||||
async function findCandidatesByVector(
|
||||
memories: Array<ExtractedMemory & { record_id: string }>,
|
||||
vectorStore: VectorStore,
|
||||
embeddingService: EmbeddingService,
|
||||
topK: number,
|
||||
logger?: Logger,
|
||||
): Promise<CandidateMatch[]> {
|
||||
const newRecordIds = new Set(memories.map((m) => m.record_id));
|
||||
|
||||
// Batch-compute embeddings for all new memories
|
||||
const texts = memories.map((m) => m.content);
|
||||
const embeddings = await embeddingService.embedBatch(texts);
|
||||
|
||||
const matches: CandidateMatch[] = [];
|
||||
|
||||
for (let i = 0; i < memories.length; i++) {
|
||||
const mem = memories[i];
|
||||
const queryVec = embeddings[i];
|
||||
|
||||
// Vector search top-K (request extra to account for self-batch filtering)
|
||||
const searchResults = vectorStore.search(queryVec, topK + memories.length);
|
||||
|
||||
// Exclude records from current batch, convert to MemoryRecord format
|
||||
const candidates: MemoryRecord[] = searchResults
|
||||
.filter((r) => !newRecordIds.has(r.record_id))
|
||||
.slice(0, topK)
|
||||
.map((r) => ({
|
||||
id: r.record_id,
|
||||
content: r.content,
|
||||
type: r.type as MemoryRecord["type"],
|
||||
priority: r.priority,
|
||||
scene_name: r.scene_name,
|
||||
source_message_ids: [],
|
||||
metadata: {},
|
||||
timestamps: [r.timestamp_str].filter(Boolean),
|
||||
createdAt: "",
|
||||
updatedAt: "",
|
||||
sessionKey: r.session_key,
|
||||
sessionId: r.session_id,
|
||||
}));
|
||||
|
||||
matches.push({ newMemory: mem, candidates });
|
||||
}
|
||||
|
||||
logger?.debug?.(
|
||||
`${TAG} Vector recall: ${matches.map((m) => `${m.newMemory.record_id}→${m.candidates.length}`).join(", ")}`,
|
||||
);
|
||||
|
||||
return matches;
|
||||
}
|
||||
|
||||
/**
|
||||
* FTS5-based candidate recall:
|
||||
* Uses the FTS index for efficient BM25-ranked keyword matching.
|
||||
* This replaces the old Jaccard word-overlap fallback entirely.
|
||||
*/
|
||||
function findCandidatesByFts(
|
||||
memories: Array<ExtractedMemory & { record_id: string }>,
|
||||
vectorStore: VectorStore,
|
||||
_logger?: Logger,
|
||||
): CandidateMatch[] {
|
||||
const newRecordIds = new Set(memories.map((m) => m.record_id));
|
||||
const matches: CandidateMatch[] = [];
|
||||
|
||||
for (const mem of memories) {
|
||||
const ftsQuery = buildFtsQuery(mem.content);
|
||||
if (ftsQuery) {
|
||||
const ftsResults = vectorStore.ftsSearchL1(ftsQuery, 10);
|
||||
// Filter out records from the current batch
|
||||
const candidates: MemoryRecord[] = ftsResults
|
||||
.filter((r) => !newRecordIds.has(r.record_id))
|
||||
.slice(0, 5)
|
||||
.map((r) => ({
|
||||
id: r.record_id,
|
||||
content: r.content,
|
||||
type: r.type as MemoryRecord["type"],
|
||||
priority: r.priority,
|
||||
scene_name: r.scene_name,
|
||||
source_message_ids: [],
|
||||
metadata: r.metadata_json ? (() => { try { return JSON.parse(r.metadata_json); } catch { return {}; } })() : {},
|
||||
timestamps: [r.timestamp_str].filter(Boolean),
|
||||
createdAt: "",
|
||||
updatedAt: "",
|
||||
sessionKey: r.session_key,
|
||||
sessionId: r.session_id,
|
||||
}));
|
||||
matches.push({ newMemory: mem, candidates });
|
||||
} else {
|
||||
matches.push({ newMemory: mem, candidates: [] });
|
||||
}
|
||||
}
|
||||
|
||||
_logger?.debug?.(`${TAG} FTS keyword recall: ${matches.map((m) => `${m.newMemory.record_id}→${m.candidates.length}`).join(", ")}`);
|
||||
return matches;
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Result parsing
|
||||
// ============================
|
||||
|
||||
const VALID_TYPES: MemoryType[] = ["persona", "episodic", "instruction"];
|
||||
|
||||
/**
|
||||
* Parse the LLM's batch conflict detection JSON response.
|
||||
*
|
||||
* Expected format: [{record_id, action, target_ids, merged_content, merged_type, merged_priority, merged_timestamps}]
|
||||
*/
|
||||
function parseBatchResult(
|
||||
raw: string,
|
||||
memories: Array<ExtractedMemory & { record_id: string }>,
|
||||
logger?: Logger,
|
||||
): DedupDecision[] {
|
||||
try {
|
||||
// Strip markdown code block wrappers
|
||||
let cleaned = raw.trim();
|
||||
if (cleaned.startsWith("```")) {
|
||||
cleaned = cleaned.replace(/^```(?:json)?\s*\n?/, "").replace(/\n?```\s*$/, "");
|
||||
}
|
||||
|
||||
// Extract JSON array
|
||||
const arrayMatch = cleaned.match(/\[[\s\S]*\]/);
|
||||
if (!arrayMatch) {
|
||||
logger?.warn?.(`${TAG} No JSON array found in conflict detection response`);
|
||||
return fallbackStoreAll(memories);
|
||||
}
|
||||
|
||||
// Sanitize control characters inside JSON string literals that LLM may produce
|
||||
const sanitized = sanitizeJsonForParse(arrayMatch[0]);
|
||||
const parsed = JSON.parse(sanitized) as unknown[];
|
||||
|
||||
if (!Array.isArray(parsed)) {
|
||||
logger?.warn?.(`${TAG} Conflict detection response is not an array`);
|
||||
return fallbackStoreAll(memories);
|
||||
}
|
||||
|
||||
// Build decisions from LLM output
|
||||
const decisions: DedupDecision[] = [];
|
||||
const validActions = ["store", "update", "merge", "skip"];
|
||||
|
||||
for (const item of parsed) {
|
||||
if (!item || typeof item !== "object") continue;
|
||||
const d = item as Record<string, unknown>;
|
||||
|
||||
const recordId = String(d.record_id ?? "");
|
||||
const action = String(d.action ?? "store");
|
||||
|
||||
if (!validActions.includes(action)) {
|
||||
logger?.warn?.(`${TAG} Invalid action "${action}" for record ${recordId}, defaulting to store`);
|
||||
}
|
||||
|
||||
decisions.push({
|
||||
record_id: recordId,
|
||||
action: validActions.includes(action) ? (action as DedupDecision["action"]) : "store",
|
||||
target_ids: Array.isArray(d.target_ids) ? d.target_ids.map(String) : [],
|
||||
merged_content: typeof d.merged_content === "string" ? d.merged_content : undefined,
|
||||
merged_type: VALID_TYPES.includes(d.merged_type as MemoryType) ? (d.merged_type as MemoryType) : undefined,
|
||||
merged_priority: typeof d.merged_priority === "number" ? d.merged_priority : undefined,
|
||||
merged_timestamps: Array.isArray(d.merged_timestamps) ? d.merged_timestamps.map(String) : undefined,
|
||||
});
|
||||
}
|
||||
|
||||
// Ensure all memories have a decision (fill missing with "store")
|
||||
const decidedIds = new Set(decisions.map((d) => d.record_id));
|
||||
for (const mem of memories) {
|
||||
if (!decidedIds.has(mem.record_id)) {
|
||||
logger?.debug?.(`${TAG} No decision for record ${mem.record_id}, defaulting to store`);
|
||||
decisions.push({
|
||||
record_id: mem.record_id,
|
||||
action: "store",
|
||||
target_ids: [],
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return decisions;
|
||||
} catch (err) {
|
||||
logger?.warn?.(`${TAG} Failed to parse conflict detection result: ${err instanceof Error ? err.message : String(err)}`);
|
||||
return fallbackStoreAll(memories);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fallback: store all memories when parsing fails.
|
||||
*/
|
||||
function fallbackStoreAll(memories: Array<ExtractedMemory & { record_id: string }>): DedupDecision[] {
|
||||
return memories.map((m) => ({
|
||||
record_id: m.record_id,
|
||||
action: "store" as const,
|
||||
target_ids: [],
|
||||
}));
|
||||
}
|
||||
@@ -0,0 +1,500 @@
|
||||
/**
|
||||
* L1 Memory Extractor: extracts structured memories from L0 conversation messages
|
||||
* using a single LLM call with JSON-mode structured output.
|
||||
*
|
||||
* v3: Aligned with Kenty's prompt — scene segmentation + memory extraction in one call,
|
||||
* followed by batch conflict detection.
|
||||
*
|
||||
* Pipeline:
|
||||
* 1. Read recent messages from L0 (split into background + new)
|
||||
* 2. Call LLM to extract scene-segmented memories
|
||||
* 3. Batch conflict detection against existing records
|
||||
* 4. Write to L1 JSONL files
|
||||
*/
|
||||
|
||||
import type { ConversationMessage } from "../conversation/l0-recorder.js";
|
||||
import { EXTRACT_MEMORIES_SYSTEM_PROMPT, formatExtractionPrompt } from "../prompts/l1-extraction.js";
|
||||
import { batchDedup } from "./l1-dedup.js";
|
||||
import { writeMemory, generateMemoryId } from "./l1-writer.js";
|
||||
import type { ExtractedMemory, MemoryRecord, MemoryType, DedupDecision } from "./l1-writer.js";
|
||||
import { CleanContextRunner } from "../utils/clean-context-runner.js";
|
||||
import { sanitizeJsonForParse, shouldExtractL1 } from "../utils/sanitize.js";
|
||||
import type { VectorStore } from "../store/vector-store.js";
|
||||
import type { EmbeddingService } from "../store/embedding.js";
|
||||
import { report } from "../report/reporter.js";
|
||||
|
||||
interface Logger {
|
||||
debug?: (message: string) => void;
|
||||
info: (message: string) => void;
|
||||
warn: (message: string) => void;
|
||||
error: (message: string) => void;
|
||||
}
|
||||
|
||||
const TAG = "[memory-tdai][l1-extractor]";
|
||||
|
||||
// ============================
|
||||
// Types
|
||||
// ============================
|
||||
|
||||
/** A scene segment with its extracted memories (LLM output) */
|
||||
interface SceneSegment {
|
||||
scene_name: string;
|
||||
message_ids: string[];
|
||||
memories: Array<{
|
||||
content: string;
|
||||
type: string;
|
||||
priority: number;
|
||||
source_message_ids: string[];
|
||||
metadata: Record<string, unknown>;
|
||||
}>;
|
||||
}
|
||||
|
||||
export interface L1ExtractionResult {
|
||||
/** Whether extraction succeeded */
|
||||
success: boolean;
|
||||
/** Number of memories extracted */
|
||||
extractedCount: number;
|
||||
/** Number of memories actually stored (after dedup) */
|
||||
storedCount: number;
|
||||
/** The memory records that were stored */
|
||||
records: MemoryRecord[];
|
||||
/** Scene names detected during extraction */
|
||||
sceneNames: string[];
|
||||
/** Last scene name (for continuity in next extraction) */
|
||||
lastSceneName?: string;
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Core function
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Run the full L1 extraction pipeline on conversation messages.
|
||||
*
|
||||
* @param messages - Filtered conversation messages (from L0 or directly from hook)
|
||||
* @param sessionKey - The session key
|
||||
* @param baseDir - Base data directory (~/.openclaw/memory-tdai/)
|
||||
* @param config - OpenClaw config (for LLM access)
|
||||
* @param options - Extraction options
|
||||
* @param logger - Optional logger
|
||||
*/
|
||||
export async function extractL1Memories(params: {
|
||||
messages: ConversationMessage[];
|
||||
sessionKey: string;
|
||||
sessionId?: string;
|
||||
baseDir: string;
|
||||
config: unknown;
|
||||
options?: {
|
||||
/** Max new messages to send in one extraction call */
|
||||
maxMessagesPerExtraction?: number;
|
||||
/** Max background messages for context */
|
||||
maxBackgroundMessages?: number;
|
||||
/** Enable conflict detection */
|
||||
enableDedup?: boolean;
|
||||
/** Max memories extracted per call */
|
||||
maxMemoriesPerSession?: number;
|
||||
/** LLM model override */
|
||||
model?: string;
|
||||
/** Previous scene name for continuity */
|
||||
previousSceneName?: string;
|
||||
/** Vector store for cosine similarity candidate recall */
|
||||
vectorStore?: VectorStore;
|
||||
/** Embedding service for computing query vectors */
|
||||
embeddingService?: EmbeddingService;
|
||||
/** Top-K candidates for conflict recall (default: 5) */
|
||||
conflictRecallTopK?: number;
|
||||
};
|
||||
logger?: Logger;
|
||||
/** Plugin instance ID for metric reporting (optional — metrics skipped if absent) */
|
||||
instanceId?: string;
|
||||
}): Promise<L1ExtractionResult> {
|
||||
const { messages, sessionKey, sessionId, baseDir, config, logger, instanceId: metricInstanceId } = params;
|
||||
const options = params.options ?? {};
|
||||
const maxNewMessages = options.maxMessagesPerExtraction ?? 10;
|
||||
const maxBgMessages = options.maxBackgroundMessages ?? 5;
|
||||
const enableDedup = options.enableDedup ?? true;
|
||||
const maxMemoriesPerSession = options.maxMemoriesPerSession ?? 10;
|
||||
|
||||
if (messages.length === 0) {
|
||||
logger?.debug?.(`${TAG} No messages to extract from`);
|
||||
return { success: true, extractedCount: 0, storedCount: 0, records: [], sceneNames: [] };
|
||||
}
|
||||
|
||||
const l1StartMs = Date.now();
|
||||
|
||||
// Quality gate: filter messages through L1 extraction rules (length, symbols,
|
||||
// prompt injection, etc.) before sending to the LLM. L0 deliberately captures
|
||||
// everything; the strict filtering happens here at L1 stage.
|
||||
const qualifiedMessages = messages.filter((m) => shouldExtractL1(m.content));
|
||||
if (qualifiedMessages.length < messages.length) {
|
||||
logger?.debug?.(
|
||||
`${TAG} L1 quality filter: ${messages.length} → ${qualifiedMessages.length} messages ` +
|
||||
`(${messages.length - qualifiedMessages.length} filtered out)`,
|
||||
);
|
||||
}
|
||||
|
||||
if (qualifiedMessages.length === 0) {
|
||||
logger?.debug?.(`${TAG} All messages filtered out by L1 quality gate`);
|
||||
return { success: true, extractedCount: 0, storedCount: 0, records: [], sceneNames: [] };
|
||||
}
|
||||
|
||||
// Split messages into background (older) + new (recent)
|
||||
const newMessages = qualifiedMessages.slice(-maxNewMessages);
|
||||
const bgEndIdx = qualifiedMessages.length - newMessages.length;
|
||||
const backgroundMessages = bgEndIdx > 0
|
||||
? qualifiedMessages.slice(Math.max(0, bgEndIdx - maxBgMessages), bgEndIdx)
|
||||
: [];
|
||||
|
||||
logger?.debug?.(`${TAG} Extracting from ${newMessages.length} new messages (+ ${backgroundMessages.length} background) [${qualifiedMessages.length} qualified from ${messages.length} input]`);
|
||||
|
||||
// Step 1: LLM extraction (scene segmentation + memory extraction)
|
||||
let scenes: SceneSegment[];
|
||||
try {
|
||||
scenes = await callLlmExtraction({
|
||||
newMessages,
|
||||
backgroundMessages,
|
||||
previousSceneName: options.previousSceneName,
|
||||
config,
|
||||
logger,
|
||||
model: options.model,
|
||||
});
|
||||
logger?.debug?.(`${TAG} LLM detected ${scenes.length} scene(s)`);
|
||||
} catch (err) {
|
||||
logger?.error(`${TAG} LLM extraction failed: ${err instanceof Error ? err.message : String(err)}`);
|
||||
return { success: false, extractedCount: 0, storedCount: 0, records: [], sceneNames: [] };
|
||||
}
|
||||
|
||||
// Flatten all memories across scenes
|
||||
const allExtracted: ExtractedMemory[] = [];
|
||||
const sceneNames: string[] = [];
|
||||
|
||||
for (const scene of scenes) {
|
||||
sceneNames.push(scene.scene_name);
|
||||
for (const mem of scene.memories) {
|
||||
const memType = normalizeType(mem.type);
|
||||
if (!memType) {
|
||||
logger?.warn?.(`${TAG} Skipping memory with invalid type "${mem.type}"`);
|
||||
continue;
|
||||
}
|
||||
allExtracted.push({
|
||||
content: mem.content,
|
||||
type: memType,
|
||||
priority: typeof mem.priority === "number" ? mem.priority : 50,
|
||||
source_message_ids: Array.isArray(mem.source_message_ids) ? mem.source_message_ids : [],
|
||||
metadata: mem.metadata ?? {},
|
||||
scene_name: scene.scene_name,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
logger?.debug?.(`${TAG} Total extracted memories: ${allExtracted.length} across ${scenes.length} scene(s)`);
|
||||
|
||||
if (allExtracted.length === 0) {
|
||||
return {
|
||||
success: true,
|
||||
extractedCount: 0,
|
||||
storedCount: 0,
|
||||
records: [],
|
||||
sceneNames,
|
||||
lastSceneName: sceneNames[sceneNames.length - 1],
|
||||
};
|
||||
}
|
||||
|
||||
// Limit per session
|
||||
let extracted = allExtracted;
|
||||
if (extracted.length > maxMemoriesPerSession) {
|
||||
logger?.debug?.(`${TAG} Limiting from ${extracted.length} to ${maxMemoriesPerSession} memories per session`);
|
||||
extracted = extracted.slice(0, maxMemoriesPerSession);
|
||||
}
|
||||
|
||||
// Assign temporary IDs to extracted memories (needed for batch dedup)
|
||||
const memoriesWithIds = extracted.map((m) => ({
|
||||
...m,
|
||||
record_id: generateMemoryId(),
|
||||
}));
|
||||
|
||||
// Step 2: Batch Conflict Detection + Write
|
||||
let storedRecords: MemoryRecord[];
|
||||
|
||||
if (enableDedup) {
|
||||
try {
|
||||
const decisions = await batchDedup({
|
||||
memories: memoriesWithIds,
|
||||
config,
|
||||
logger,
|
||||
model: options.model,
|
||||
vectorStore: options.vectorStore,
|
||||
embeddingService: options.embeddingService,
|
||||
conflictRecallTopK: options.conflictRecallTopK,
|
||||
});
|
||||
|
||||
storedRecords = await applyDecisions({
|
||||
memoriesWithIds,
|
||||
decisions,
|
||||
baseDir,
|
||||
sessionKey,
|
||||
sessionId,
|
||||
logger,
|
||||
vectorStore: options.vectorStore,
|
||||
embeddingService: options.embeddingService,
|
||||
});
|
||||
} catch (err) {
|
||||
logger?.warn?.(`${TAG} Batch dedup failed, storing all as new: ${err instanceof Error ? err.message : String(err)}`);
|
||||
storedRecords = await storeAllDirectly(memoriesWithIds, baseDir, sessionKey, sessionId, logger, options.vectorStore, options.embeddingService);
|
||||
}
|
||||
} else {
|
||||
storedRecords = await storeAllDirectly(memoriesWithIds, baseDir, sessionKey, sessionId, logger, options.vectorStore, options.embeddingService);
|
||||
}
|
||||
|
||||
logger?.info(`${TAG} Extraction complete: extracted=${extracted.length}, stored=${storedRecords.length}`);
|
||||
|
||||
// ── l1_extraction metric ──
|
||||
if (metricInstanceId && logger) {
|
||||
// Build type distribution of stored memories
|
||||
const memoriesByType: Record<string, number> = {};
|
||||
for (const r of storedRecords) {
|
||||
memoriesByType[r.type] = (memoriesByType[r.type] ?? 0) + 1;
|
||||
}
|
||||
report("l1_extraction", {
|
||||
sessionKey,
|
||||
inputMessageCount: messages.length,
|
||||
memoriesExtracted: extracted.length,
|
||||
memoriesStored: storedRecords.length,
|
||||
memoriesStoredContent: storedRecords.map((r) => ({
|
||||
content: r.content,
|
||||
type: r.type,
|
||||
scene: r.scene_name ?? null,
|
||||
})),
|
||||
memoriesByType,
|
||||
totalDurationMs: Date.now() - l1StartMs,
|
||||
success: true,
|
||||
error: null,
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
extractedCount: extracted.length,
|
||||
storedCount: storedRecords.length,
|
||||
records: storedRecords,
|
||||
sceneNames,
|
||||
lastSceneName: sceneNames[sceneNames.length - 1],
|
||||
};
|
||||
}
|
||||
|
||||
// ============================
|
||||
// LLM call
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Call LLM to extract scene-segmented memories from conversation messages.
|
||||
*/
|
||||
async function callLlmExtraction(params: {
|
||||
newMessages: ConversationMessage[];
|
||||
backgroundMessages: ConversationMessage[];
|
||||
previousSceneName?: string;
|
||||
config: unknown;
|
||||
logger?: Logger;
|
||||
model?: string;
|
||||
}): Promise<SceneSegment[]> {
|
||||
const { newMessages, backgroundMessages, previousSceneName, config, logger, model } = params;
|
||||
|
||||
const runner = new CleanContextRunner({
|
||||
config,
|
||||
modelRef: model,
|
||||
enableTools: false,
|
||||
logger,
|
||||
});
|
||||
|
||||
const userPrompt = formatExtractionPrompt({
|
||||
newMessages,
|
||||
backgroundMessages,
|
||||
previousSceneName,
|
||||
});
|
||||
|
||||
const result = await runner.run({
|
||||
prompt: userPrompt,
|
||||
systemPrompt: EXTRACT_MEMORIES_SYSTEM_PROMPT,
|
||||
taskId: "l1-extraction",
|
||||
timeoutMs: 180_000,
|
||||
// maxTokens: 4000,
|
||||
});
|
||||
|
||||
return parseExtractionResult(result, logger);
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse the LLM's JSON response into SceneSegment array.
|
||||
* Expected format: [{scene_name, message_ids, memories: [...]}]
|
||||
*/
|
||||
function parseExtractionResult(raw: string, logger?: Logger): SceneSegment[] {
|
||||
try {
|
||||
// Strip markdown code block wrappers if present
|
||||
let cleaned = raw.trim();
|
||||
if (cleaned.startsWith("```")) {
|
||||
cleaned = cleaned.replace(/^```(?:json)?\s*\n?/, "").replace(/\n?```\s*$/, "");
|
||||
}
|
||||
|
||||
// Try to extract JSON array
|
||||
const arrayMatch = cleaned.match(/\[[\s\S]*\]/);
|
||||
if (!arrayMatch) {
|
||||
logger?.warn?.(`${TAG} No JSON array found in extraction response`);
|
||||
return [];
|
||||
}
|
||||
|
||||
// Sanitize control characters inside JSON string literals that LLM may produce
|
||||
const sanitized = sanitizeJsonForParse(arrayMatch[0]);
|
||||
const parsed = JSON.parse(sanitized) as unknown[];
|
||||
|
||||
if (!Array.isArray(parsed)) {
|
||||
logger?.warn?.(`${TAG} Extraction response is not an array`);
|
||||
return [];
|
||||
}
|
||||
|
||||
const scenes: SceneSegment[] = [];
|
||||
for (const item of parsed) {
|
||||
if (!item || typeof item !== "object") continue;
|
||||
const s = item as Record<string, unknown>;
|
||||
|
||||
scenes.push({
|
||||
scene_name: typeof s.scene_name === "string" ? s.scene_name : "未知情境",
|
||||
message_ids: Array.isArray(s.message_ids) ? s.message_ids.map(String) : [],
|
||||
memories: Array.isArray(s.memories)
|
||||
? (s.memories as Array<Record<string, unknown>>)
|
||||
.filter((m) => m && typeof m === "object" && typeof m.content === "string" && (m.content as string).length > 0)
|
||||
.map((m) => ({
|
||||
content: String(m.content),
|
||||
type: String(m.type ?? "episodic"),
|
||||
priority: typeof m.priority === "number" ? m.priority : 50,
|
||||
source_message_ids: Array.isArray(m.source_message_ids) ? m.source_message_ids.map(String) : [],
|
||||
metadata: (m.metadata && typeof m.metadata === "object" ? m.metadata : {}) as Record<string, unknown>,
|
||||
}))
|
||||
: [],
|
||||
});
|
||||
}
|
||||
|
||||
return scenes;
|
||||
} catch (err) {
|
||||
logger?.warn?.(`${TAG} Failed to parse extraction result: ${err instanceof Error ? err.message : String(err)}`);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Write helpers
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Apply batch dedup decisions — write memories according to their decisions.
|
||||
*/
|
||||
async function applyDecisions(params: {
|
||||
memoriesWithIds: Array<ExtractedMemory & { record_id: string }>;
|
||||
decisions: DedupDecision[];
|
||||
baseDir: string;
|
||||
sessionKey: string;
|
||||
sessionId?: string;
|
||||
logger?: Logger;
|
||||
vectorStore?: VectorStore;
|
||||
embeddingService?: EmbeddingService;
|
||||
}): Promise<MemoryRecord[]> {
|
||||
const { memoriesWithIds, decisions, baseDir, sessionKey, sessionId, logger, vectorStore, embeddingService } = params;
|
||||
const storedRecords: MemoryRecord[] = [];
|
||||
|
||||
// Build a map from record_id → decision
|
||||
const decisionMap = new Map<string, DedupDecision>();
|
||||
for (const d of decisions) {
|
||||
decisionMap.set(d.record_id, d);
|
||||
}
|
||||
|
||||
for (const memoryWithId of memoriesWithIds) {
|
||||
const decision = decisionMap.get(memoryWithId.record_id) ?? {
|
||||
record_id: memoryWithId.record_id,
|
||||
action: "store" as const,
|
||||
target_ids: [],
|
||||
};
|
||||
|
||||
try {
|
||||
const record = await writeMemory({
|
||||
memory: memoryWithId,
|
||||
decision,
|
||||
baseDir,
|
||||
sessionKey,
|
||||
sessionId,
|
||||
logger,
|
||||
vectorStore,
|
||||
embeddingService,
|
||||
});
|
||||
|
||||
if (record) {
|
||||
storedRecords.push(record);
|
||||
}
|
||||
} catch (err) {
|
||||
logger?.warn?.(
|
||||
`${TAG} Write failed for memory "${memoryWithId.content.slice(0, 50)}...": ${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return storedRecords;
|
||||
}
|
||||
|
||||
/**
|
||||
* Store all memories directly (no dedup).
|
||||
*/
|
||||
async function storeAllDirectly(
|
||||
memoriesWithIds: Array<ExtractedMemory & { record_id: string }>,
|
||||
baseDir: string,
|
||||
sessionKey: string,
|
||||
sessionId: string | undefined,
|
||||
logger?: Logger,
|
||||
vectorStore?: VectorStore,
|
||||
embeddingService?: EmbeddingService,
|
||||
): Promise<MemoryRecord[]> {
|
||||
const storedRecords: MemoryRecord[] = [];
|
||||
|
||||
for (const memoryWithId of memoriesWithIds) {
|
||||
try {
|
||||
const record = await writeMemory({
|
||||
memory: memoryWithId,
|
||||
decision: {
|
||||
record_id: memoryWithId.record_id,
|
||||
action: "store",
|
||||
target_ids: [],
|
||||
},
|
||||
baseDir,
|
||||
sessionKey,
|
||||
sessionId,
|
||||
logger,
|
||||
vectorStore,
|
||||
embeddingService,
|
||||
});
|
||||
if (record) {
|
||||
storedRecords.push(record);
|
||||
}
|
||||
} catch (err) {
|
||||
logger?.warn?.(
|
||||
`${TAG} Write failed for memory "${memoryWithId.content.slice(0, 50)}...": ${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return storedRecords;
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Helpers
|
||||
// ============================
|
||||
|
||||
const VALID_TYPES: MemoryType[] = ["persona", "episodic", "instruction"];
|
||||
|
||||
function normalizeType(raw: string): MemoryType | null {
|
||||
const lower = raw.toLowerCase().trim();
|
||||
if (VALID_TYPES.includes(lower as MemoryType)) {
|
||||
return lower as MemoryType;
|
||||
}
|
||||
// Handle legacy type names
|
||||
if (lower === "episode") return "episodic";
|
||||
if (lower === "instruct") return "instruction";
|
||||
if (lower === "preference") return "persona"; // fold preference into persona
|
||||
return null;
|
||||
}
|
||||
@@ -0,0 +1,218 @@
|
||||
/**
|
||||
* L1 Memory Reader: reads persisted L1 memory records.
|
||||
*
|
||||
* Provides two data paths:
|
||||
*
|
||||
* 1. **SQLite** (preferred): `queryMemoryRecords()` — uses VectorStore's `queryL1Records()`
|
||||
* with composite indexes on (session_key, updated_time) and (session_id, updated_time)
|
||||
* for efficient session-scoped and time-range queries.
|
||||
*
|
||||
* 2. **JSONL** (fallback): `readMemoryRecords()` / `readAllMemoryRecords()` — reads from
|
||||
* `records/YYYY-MM-DD.jsonl` files. Used when VectorStore is unavailable or degraded.
|
||||
*/
|
||||
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import type { MemoryRecord, MemoryType, EpisodicMetadata } from "./l1-writer.js";
|
||||
import type { VectorStore, L1RecordRow, L1QueryFilter } from "../store/vector-store.js";
|
||||
|
||||
// Re-export types that readers need
|
||||
export type { MemoryRecord, MemoryType, EpisodicMetadata } from "./l1-writer.js";
|
||||
export type { L1QueryFilter } from "../store/vector-store.js";
|
||||
|
||||
interface Logger {
|
||||
debug?: (message: string) => void;
|
||||
info: (message: string) => void;
|
||||
warn: (message: string) => void;
|
||||
error: (message: string) => void;
|
||||
}
|
||||
|
||||
const TAG = "[memory-tdai] [l1-reader]";
|
||||
|
||||
// ============================
|
||||
// SQLite-based queries (preferred)
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Query L1 memory records from SQLite via VectorStore.
|
||||
*
|
||||
* This is the **preferred** read path — it uses the composite index
|
||||
* `idx_l1_session_updated(session_id, updated_time)` for efficient
|
||||
* session-scoped and time-range queries.
|
||||
*
|
||||
* All timestamps are UTC ISO 8601 (as stored by l1-writer's dual-write).
|
||||
*
|
||||
* Falls back to empty array if VectorStore is null or degraded.
|
||||
*/
|
||||
export function queryMemoryRecords(
|
||||
vectorStore: VectorStore | null | undefined,
|
||||
filter?: L1QueryFilter,
|
||||
logger?: Logger,
|
||||
): MemoryRecord[] {
|
||||
if (!vectorStore) {
|
||||
logger?.warn(`${TAG} queryMemoryRecords: no VectorStore available, returning empty`);
|
||||
return [];
|
||||
}
|
||||
|
||||
const rows = vectorStore.queryL1Records(filter);
|
||||
return rows.map(rowToMemoryRecord);
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a raw SQLite L1RecordRow to a MemoryRecord (same shape as JSONL records).
|
||||
*/
|
||||
function rowToMemoryRecord(row: L1RecordRow): MemoryRecord {
|
||||
let metadata: EpisodicMetadata | Record<string, never> = {};
|
||||
try {
|
||||
metadata = JSON.parse(row.metadata_json) as EpisodicMetadata | Record<string, never>;
|
||||
} catch {
|
||||
// malformed JSON — use empty object
|
||||
}
|
||||
|
||||
// Reconstruct timestamps array from timestamp_start / timestamp_end
|
||||
const timestamps: string[] = [];
|
||||
if (row.timestamp_str) timestamps.push(row.timestamp_str);
|
||||
if (row.timestamp_start && row.timestamp_start !== row.timestamp_str) timestamps.push(row.timestamp_start);
|
||||
if (row.timestamp_end && row.timestamp_end !== row.timestamp_str && row.timestamp_end !== row.timestamp_start) {
|
||||
timestamps.push(row.timestamp_end);
|
||||
}
|
||||
|
||||
return {
|
||||
id: row.record_id,
|
||||
content: row.content,
|
||||
type: row.type as MemoryType,
|
||||
priority: row.priority,
|
||||
scene_name: row.scene_name,
|
||||
source_message_ids: [], // not stored in SQLite (vector search doesn't need them)
|
||||
metadata,
|
||||
timestamps,
|
||||
createdAt: row.created_time,
|
||||
updatedAt: row.updated_time,
|
||||
sessionKey: row.session_key,
|
||||
sessionId: row.session_id,
|
||||
};
|
||||
}
|
||||
|
||||
// ============================
|
||||
// JSONL-based reads (fallback)
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Read all memory records for a session from JSONL files.
|
||||
*
|
||||
* Current naming mode:
|
||||
* - Daily merged file: records/YYYY-MM-DD.jsonl (all sessions in one file)
|
||||
*/
|
||||
export async function readMemoryRecords(
|
||||
sessionKey: string,
|
||||
baseDir: string,
|
||||
logger?: Logger,
|
||||
): Promise<MemoryRecord[]> {
|
||||
const recordsDir = path.join(baseDir, "records");
|
||||
const dateFilePattern = /^\d{4}-\d{2}-\d{2}\.jsonl$/;
|
||||
|
||||
let entries: import("node:fs").Dirent[];
|
||||
try {
|
||||
entries = await fs.readdir(recordsDir, { withFileTypes: true });
|
||||
} catch {
|
||||
// Directory doesn't exist yet
|
||||
return [];
|
||||
}
|
||||
|
||||
const targetFiles = entries
|
||||
.filter((entry) => entry.isFile() && dateFilePattern.test(entry.name))
|
||||
.map((entry) => entry.name)
|
||||
.sort();
|
||||
|
||||
if (targetFiles.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const records: MemoryRecord[] = [];
|
||||
|
||||
for (const fileName of targetFiles) {
|
||||
const filePath = path.join(recordsDir, fileName);
|
||||
|
||||
let raw: string;
|
||||
try {
|
||||
raw = await fs.readFile(filePath, "utf-8");
|
||||
} catch {
|
||||
logger?.warn?.(`${TAG} Failed to read L1 file: ${filePath}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const lines = raw.split("\n").filter((line) => line.trim());
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const line = lines[i];
|
||||
try {
|
||||
const parsed = JSON.parse(line) as Partial<MemoryRecord>;
|
||||
if (parsed.sessionKey !== sessionKey) {
|
||||
continue;
|
||||
}
|
||||
records.push(parsed as MemoryRecord);
|
||||
} catch {
|
||||
logger?.warn?.(`${TAG} Skipping malformed JSONL line in ${filePath}:${i + 1}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
records.sort((a, b) => {
|
||||
const ta = a.updatedAt || a.createdAt || "";
|
||||
const tb = b.updatedAt || b.createdAt || "";
|
||||
return ta.localeCompare(tb);
|
||||
});
|
||||
|
||||
return records;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read ALL memory records across all session JSONL files.
|
||||
*/
|
||||
export async function readAllMemoryRecords(
|
||||
baseDir: string,
|
||||
logger?: Logger,
|
||||
): Promise<MemoryRecord[]> {
|
||||
const recordsDir = path.join(baseDir, "records");
|
||||
try {
|
||||
const files = await fs.readdir(recordsDir);
|
||||
const allRecords: MemoryRecord[] = [];
|
||||
|
||||
for (const file of files) {
|
||||
if (!file.endsWith(".jsonl")) continue;
|
||||
const filePath = path.join(recordsDir, file);
|
||||
try {
|
||||
const raw = await fs.readFile(filePath, "utf-8");
|
||||
const lines = raw.split("\n").filter((line: string) => line.trim());
|
||||
for (const line of lines) {
|
||||
try {
|
||||
allRecords.push(JSON.parse(line) as MemoryRecord);
|
||||
} catch {
|
||||
logger?.warn?.(`${TAG} Skipping malformed JSONL line in ${file}`);
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
logger?.warn?.(`${TAG} Failed to read ${file}`);
|
||||
}
|
||||
}
|
||||
|
||||
allRecords.sort((a, b) => {
|
||||
const ta = a.updatedAt || a.createdAt || "";
|
||||
const tb = b.updatedAt || b.createdAt || "";
|
||||
return ta.localeCompare(tb);
|
||||
});
|
||||
|
||||
return allRecords;
|
||||
|
||||
} catch {
|
||||
// records/ directory doesn't exist yet
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Helpers
|
||||
// ============================
|
||||
|
||||
function sanitizeFilename(name: string): string {
|
||||
return name.replace(/[<>:"/\\|?*\x00-\x1f]/g, "_");
|
||||
}
|
||||
@@ -0,0 +1,280 @@
|
||||
/**
|
||||
* L1 Memory Writer: writes extracted memories to JSONL files.
|
||||
*
|
||||
* File naming: records/YYYY-MM-DD.jsonl (daily shards, all sessions merged).
|
||||
* Each record includes sessionKey for traceability.
|
||||
*
|
||||
* Write strategy:
|
||||
* - JSONL is the append-only persistent store (source of truth for backup/recovery).
|
||||
* - VectorStore (SQLite) is the primary retrieval engine.
|
||||
* - On update/merge, old records are deleted from VectorStore in real-time;
|
||||
* JSONL is append-only and cleaned up periodically by memory-cleaner.
|
||||
*
|
||||
* Supports store (append), update, merge, and skip operations.
|
||||
*
|
||||
* v3: Aligned with Kenty's prompt output format — 3 memory types (persona/episodic/instruction),
|
||||
* numeric priority, scene_name, source_message_ids, metadata, timestamps.
|
||||
*/
|
||||
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import crypto from "node:crypto";
|
||||
import type { VectorStore } from "../store/vector-store.js";
|
||||
import type { EmbeddingService } from "../store/embedding.js";
|
||||
|
||||
// ============================
|
||||
// Types
|
||||
// ============================
|
||||
|
||||
/** v3: 3 memory types aligned with Kenty's extraction prompt */
|
||||
export type MemoryType = "persona" | "episodic" | "instruction";
|
||||
|
||||
/** Metadata for episodic memories (activity time range) */
|
||||
export interface EpisodicMetadata {
|
||||
activity_start_time?: string; // ISO 8601
|
||||
activity_end_time?: string; // ISO 8601
|
||||
}
|
||||
|
||||
/**
|
||||
* A persisted memory record in L1 JSONL files.
|
||||
*
|
||||
* v3 changes from v2:
|
||||
* - `importance: "high"|"medium"|"low"` → `priority: number` (0-100, -1 for strict global instructions)
|
||||
* - Added `scene_name`, `source_message_ids`, `metadata`, `timestamps`
|
||||
* - Removed `keywords` (will be rebuilt from content for search)
|
||||
* - MemoryType reduced from 4 to 3 (removed "preference", folded into "persona")
|
||||
*/
|
||||
export interface MemoryRecord {
|
||||
/** Unique ID for dedup updates */
|
||||
id: string;
|
||||
/** Memory content */
|
||||
content: string;
|
||||
/** Memory type: persona / episodic / instruction */
|
||||
type: MemoryType;
|
||||
/** Priority score: 0-100 (higher = more important), -1 = strict global instruction */
|
||||
priority: number;
|
||||
/** Scene name this memory belongs to */
|
||||
scene_name: string;
|
||||
/** Source message IDs that contributed to this memory */
|
||||
source_message_ids: string[];
|
||||
/** Type-specific metadata (e.g., activity_start_time for episodic) */
|
||||
metadata: EpisodicMetadata | Record<string, never>;
|
||||
/** Timestamp trail: all timestamps related to this memory (for merge history tracking) */
|
||||
timestamps: string[];
|
||||
/** Creation timestamp (ISO) */
|
||||
createdAt: string;
|
||||
/** Last update timestamp (ISO) */
|
||||
updatedAt: string;
|
||||
/** Source session key (conversation channel identifier) */
|
||||
sessionKey: string;
|
||||
/** Source session ID (single conversation instance identifier) */
|
||||
sessionId: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* A memory as extracted by LLM (before dedup / persistence).
|
||||
* Matches the output format of Kenty's extraction prompt.
|
||||
*/
|
||||
export interface ExtractedMemory {
|
||||
content: string;
|
||||
type: MemoryType;
|
||||
priority: number;
|
||||
source_message_ids: string[];
|
||||
metadata: EpisodicMetadata | Record<string, never>;
|
||||
/** Scene name this memory was extracted in */
|
||||
scene_name: string;
|
||||
}
|
||||
|
||||
export type DedupAction = "store" | "update" | "merge" | "skip";
|
||||
|
||||
/**
|
||||
* v3 batch dedup decision — one per new memory, aligned with Kenty's conflict detection prompt.
|
||||
*
|
||||
* Key changes:
|
||||
* - `targetId` → `target_ids` (array, supports multi-target merge/update)
|
||||
* - Added `merged_type`, `merged_priority`, `merged_timestamps` for cross-type merge
|
||||
*/
|
||||
export interface DedupDecision {
|
||||
/** Which new memory this decision is about */
|
||||
record_id: string;
|
||||
action: DedupAction;
|
||||
/** IDs of existing records to replace/remove (for update/merge) */
|
||||
target_ids: string[];
|
||||
/** Merged/updated content text (for update/merge) */
|
||||
merged_content?: string;
|
||||
/** Best type after merge (for update/merge, may differ from original) */
|
||||
merged_type?: MemoryType;
|
||||
/** Priority after merge (for update/merge) */
|
||||
merged_priority?: number;
|
||||
/** Union of all related timestamps (for update/merge) */
|
||||
merged_timestamps?: string[];
|
||||
}
|
||||
|
||||
interface Logger {
|
||||
debug?: (message: string) => void;
|
||||
info: (message: string) => void;
|
||||
warn: (message: string) => void;
|
||||
error: (message: string) => void;
|
||||
}
|
||||
|
||||
const TAG = "[memory-tdai][l1-writer]";
|
||||
|
||||
// ============================
|
||||
// Core functions
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Generate a unique memory ID.
|
||||
*/
|
||||
export function generateMemoryId(): string {
|
||||
return `m_${Date.now()}_${crypto.randomBytes(4).toString("hex")}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Write a memory record according to the dedup decision.
|
||||
*
|
||||
* - store: append new record
|
||||
* - update: remove target records + append updated record
|
||||
* - merge: remove target records + append merged record
|
||||
* - skip: do nothing
|
||||
*
|
||||
* v3: supports multi-target removal for update/merge.
|
||||
* v3.1: optional VectorStore + EmbeddingService for dual-write (JSONL + vector).
|
||||
*/
|
||||
export async function writeMemory(params: {
|
||||
memory: ExtractedMemory;
|
||||
decision: DedupDecision;
|
||||
baseDir: string;
|
||||
sessionKey: string;
|
||||
sessionId?: string;
|
||||
logger?: Logger;
|
||||
/** Optional vector store for dual-write (JSONL + vector DB) */
|
||||
vectorStore?: VectorStore;
|
||||
/** Optional embedding service (required when vectorStore is provided) */
|
||||
embeddingService?: EmbeddingService;
|
||||
}): Promise<MemoryRecord | null> {
|
||||
const { memory, decision, baseDir, sessionKey, sessionId, logger, vectorStore, embeddingService } = params;
|
||||
|
||||
if (decision.action === "skip") {
|
||||
logger?.debug?.(`${TAG} Skipping memory: ${memory.content.slice(0, 50)}...`);
|
||||
return null;
|
||||
}
|
||||
|
||||
const now = new Date().toISOString();
|
||||
|
||||
// Determine final content, type, priority based on action
|
||||
let finalContent: string;
|
||||
let finalType: MemoryType;
|
||||
let finalPriority: number;
|
||||
let finalTimestamps: string[];
|
||||
|
||||
if (decision.action === "merge" || decision.action === "update") {
|
||||
finalContent = decision.merged_content ?? memory.content;
|
||||
finalType = decision.merged_type ?? memory.type;
|
||||
finalPriority = decision.merged_priority ?? memory.priority;
|
||||
finalTimestamps = decision.merged_timestamps ?? [now];
|
||||
} else {
|
||||
// store
|
||||
finalContent = memory.content;
|
||||
finalType = memory.type;
|
||||
finalPriority = memory.priority;
|
||||
finalTimestamps = [now];
|
||||
}
|
||||
|
||||
const record: MemoryRecord = {
|
||||
id: decision.record_id || generateMemoryId(),
|
||||
content: finalContent,
|
||||
type: finalType,
|
||||
priority: finalPriority,
|
||||
scene_name: memory.scene_name,
|
||||
source_message_ids: memory.source_message_ids,
|
||||
metadata: memory.metadata,
|
||||
timestamps: finalTimestamps,
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
sessionKey,
|
||||
sessionId: sessionId || "",
|
||||
};
|
||||
|
||||
const recordsDir = path.join(baseDir, "records");
|
||||
await fs.mkdir(recordsDir, { recursive: true });
|
||||
|
||||
const shardDate = formatLocalDate(new Date());
|
||||
const filePath = path.join(recordsDir, `${shardDate}.jsonl`);
|
||||
|
||||
if ((decision.action === "update" || decision.action === "merge") && decision.target_ids.length > 0) {
|
||||
// Remove target records from VectorStore (real-time deletion for retrieval accuracy).
|
||||
// JSONL is append-only — old records remain in files and are cleaned up periodically
|
||||
// by memory-cleaner (which reconciles against VectorStore as source of truth).
|
||||
if (vectorStore) {
|
||||
try {
|
||||
vectorStore.deleteBatch(decision.target_ids);
|
||||
logger?.debug?.(`${TAG} VectorStore: deleted ${decision.target_ids.length} target record(s) for ${decision.action}`);
|
||||
} catch (err) {
|
||||
logger?.warn?.(
|
||||
`${TAG} VectorStore delete failed for ${decision.action}: ${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
await fs.appendFile(filePath, JSON.stringify(record) + "\n", "utf-8");
|
||||
logger?.debug?.(`${TAG} ${decision.action} memory: removed [${decision.target_ids.join(",")}] from VectorStore → ${record.id}: ${finalContent.slice(0, 80)}...`);
|
||||
} else {
|
||||
// store: append a new line
|
||||
await fs.appendFile(filePath, JSON.stringify(record) + "\n", "utf-8");
|
||||
logger?.debug?.(`${TAG} Stored memory ${record.id}: ${finalContent.slice(0, 80)}...`);
|
||||
}
|
||||
|
||||
// === Vector Store dual-write ===
|
||||
if (vectorStore) {
|
||||
try {
|
||||
logger?.debug?.(
|
||||
`${TAG} [vec-dual-write] START id=${record.id}, contentLen=${record.content.length}, ` +
|
||||
`content="${record.content.slice(0, 80)}..."`,
|
||||
);
|
||||
|
||||
let embedding: Float32Array | undefined;
|
||||
|
||||
if (embeddingService) {
|
||||
try {
|
||||
embedding = await embeddingService.embed(record.content);
|
||||
logger?.debug?.(
|
||||
`${TAG} [vec-dual-write] Embedding OK: dims=${embedding.length}, ` +
|
||||
`norm=${Math.sqrt(Array.from(embedding).reduce((s, v) => s + v * v, 0)).toFixed(4)}`,
|
||||
);
|
||||
} catch (embedErr) {
|
||||
// Embedding failed — pass undefined to upsert() which writes
|
||||
// metadata + FTS only, skipping the vec0 table.
|
||||
logger?.warn(
|
||||
`${TAG} [vec-dual-write] Embedding FAILED for id=${record.id}, ` +
|
||||
`will write metadata only: ${embedErr instanceof Error ? embedErr.message : String(embedErr)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const upsertOk = vectorStore.upsert(record, embedding);
|
||||
logger?.debug?.(`${TAG} [vec-dual-write] upsert result=${upsertOk} id=${record.id}`);
|
||||
} catch (err) {
|
||||
// Vector write failure should NOT block the main JSONL write
|
||||
logger?.warn?.(
|
||||
`${TAG} [vec-dual-write] FAILED (JSONL already written) id=${record.id}: ${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
}
|
||||
} else {
|
||||
logger?.debug?.(
|
||||
`${TAG} [vec-dual-write] SKIPPED id=${record.id}: vectorStore=${!!vectorStore}`,
|
||||
);
|
||||
}
|
||||
|
||||
return record;
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Helpers
|
||||
// ============================
|
||||
|
||||
function formatLocalDate(d: Date): string {
|
||||
const y = d.getFullYear();
|
||||
const m = String(d.getMonth() + 1).padStart(2, "0");
|
||||
const day = String(d.getDate()).padStart(2, "0");
|
||||
return `${y}-${m}-${day}`;
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
|
||||
export const REPORT_CONST = {
|
||||
PLUGIN: "plugin",
|
||||
} as const;
|
||||
|
||||
export type ReportPayload = Record<string, unknown>;
|
||||
|
||||
export interface IReporter {
|
||||
reportFunc(category: string, payload: ReportPayload): void;
|
||||
}
|
||||
|
||||
// ── Singleton ──
|
||||
|
||||
let _reporter: IReporter | undefined;
|
||||
|
||||
export function initReporter(opts: {
|
||||
enabled: boolean;
|
||||
type: string;
|
||||
logger: { info: (msg: string) => void };
|
||||
instanceId: string;
|
||||
pluginVersion: string;
|
||||
}): void {
|
||||
if (_reporter) return;
|
||||
if (!opts.enabled) return;
|
||||
switch (opts.type) {
|
||||
case "local":
|
||||
_reporter = new LocalReporter(opts.logger, opts.instanceId, opts.pluginVersion);
|
||||
break;
|
||||
// TODO: add new reporter type
|
||||
default:
|
||||
opts.logger.info(`[memory-tdai] Unknown reporter type "${opts.type}", disabled reporting`);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
export function setReporter(reporter: IReporter): void {
|
||||
_reporter = reporter;
|
||||
}
|
||||
|
||||
export function report(event: string, data: ReportPayload): void {
|
||||
if (!_reporter) return;
|
||||
try {
|
||||
_reporter.reportFunc(REPORT_CONST.PLUGIN, { event, ...data });
|
||||
} catch { /* never block business logic */ }
|
||||
}
|
||||
|
||||
// ── LocalReporter (default) ──
|
||||
|
||||
class LocalReporter implements IReporter {
|
||||
constructor(
|
||||
private readonly logger: { info: (msg: string) => void },
|
||||
private readonly instanceId: string,
|
||||
private readonly pluginVersion: string,
|
||||
) {}
|
||||
|
||||
reportFunc(category: string, payload: ReportPayload): void {
|
||||
try {
|
||||
this.logger.info(JSON.stringify({
|
||||
tag: "METRIC",
|
||||
category,
|
||||
plugin: "memory-tdai",
|
||||
instanceId: this.instanceId,
|
||||
pluginVersion: this.pluginVersion,
|
||||
ts: new Date().toISOString(),
|
||||
...payload,
|
||||
}));
|
||||
} catch { /* swallow */ }
|
||||
}
|
||||
}
|
||||
|
||||
// ── Instance ID (persisted per-install) ──
|
||||
|
||||
let _instanceIdCache: string | undefined;
|
||||
|
||||
export async function getOrCreateInstanceId(pluginDataDir: string): Promise<string> {
|
||||
if (_instanceIdCache) return _instanceIdCache;
|
||||
|
||||
const idFile = path.join(pluginDataDir, ".metadata", "instance_id");
|
||||
try {
|
||||
const existing = (await fs.readFile(idFile, "utf-8")).trim();
|
||||
if (existing) {
|
||||
_instanceIdCache = existing;
|
||||
return existing;
|
||||
}
|
||||
} catch { /* file doesn't exist */ }
|
||||
|
||||
const newId = randomUUID();
|
||||
await fs.mkdir(path.dirname(idFile), { recursive: true });
|
||||
await fs.writeFile(idFile, newId, "utf-8");
|
||||
_instanceIdCache = newId;
|
||||
return newId;
|
||||
}
|
||||
@@ -0,0 +1,416 @@
|
||||
/**
|
||||
* SceneExtractor: LLM-driven memory extraction into scene blocks.
|
||||
*
|
||||
* Replaces the keyword-based SceneManager.processNewMemories() with an
|
||||
* LLM agent that autonomously reads/writes scene block files using tools.
|
||||
*
|
||||
* Security: The LLM is sandboxed — workspaceDir is set to scene_blocks/
|
||||
* so it can ONLY operate on .md scene files. System files (checkpoint,
|
||||
* scene_index, persona.md) are physically invisible to the LLM.
|
||||
*
|
||||
* Flow:
|
||||
* 1. Backup + load scene index + build summaries
|
||||
* 2. Assemble extraction prompt with memories + scene context
|
||||
* 3. Run via CleanContextRunner (tools enabled, sandboxed to scene_blocks/)
|
||||
* 4. Cleanup: remove soft-deletes, sync index, update navigation
|
||||
* 5. Parse LLM text output for out-of-band persona update signals
|
||||
*/
|
||||
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { CleanContextRunner } from "../utils/clean-context-runner.js";
|
||||
import { CheckpointManager } from "../utils/checkpoint.js";
|
||||
import { BackupManager } from "../utils/backup.js";
|
||||
import { readSceneIndex, syncSceneIndex } from "../scene/scene-index.js";
|
||||
import type { SceneIndexEntry } from "../scene/scene-index.js";
|
||||
import { parseSceneBlock } from "../scene/scene-format.js";
|
||||
import { generateSceneNavigation, stripSceneNavigation } from "../scene/scene-navigation.js";
|
||||
import { buildSceneExtractionPrompt } from "../prompts/scene-extraction.js";
|
||||
import { report } from "../report/reporter.js";
|
||||
|
||||
const TAG = "[memory-tdai] [extractor]";
|
||||
|
||||
interface ExtractorLogger {
|
||||
debug?: (message: string) => void;
|
||||
info: (message: string) => void;
|
||||
warn: (message: string) => void;
|
||||
error: (message: string) => void;
|
||||
}
|
||||
|
||||
export interface ExtractionResult {
|
||||
memoriesProcessed: number;
|
||||
success: boolean;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export interface SceneExtractorOptions {
|
||||
dataDir: string;
|
||||
config: unknown;
|
||||
model?: string;
|
||||
maxScenes?: number;
|
||||
sceneBackupCount?: number;
|
||||
timeoutMs?: number;
|
||||
logger?: ExtractorLogger;
|
||||
/** Plugin instance ID for metric reporting (optional) */
|
||||
instanceId?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse LLM text output for a persona update request signal.
|
||||
*
|
||||
* Supports multiple formats for robustness:
|
||||
* - Block: [PERSONA_UPDATE_REQUEST]reason: xxx[/PERSONA_UPDATE_REQUEST]
|
||||
* - Inline: PERSONA_UPDATE_REQUEST: xxx
|
||||
*/
|
||||
export function parsePersonaUpdateSignal(text: string): { reason: string } | null {
|
||||
// Block format: [PERSONA_UPDATE_REQUEST]...[/PERSONA_UPDATE_REQUEST]
|
||||
const blockMatch = text.match(
|
||||
/\[PERSONA_UPDATE_REQUEST\]\s*(?:reason:\s*)?(.+?)\s*\[\/PERSONA_UPDATE_REQUEST\]/s,
|
||||
);
|
||||
if (blockMatch) return { reason: blockMatch[1]!.trim() };
|
||||
|
||||
// Inline format: PERSONA_UPDATE_REQUEST: reason text
|
||||
const inlineMatch = text.match(
|
||||
/PERSONA_UPDATE_REQUEST:\s*(.+?)(?:\n|$)/,
|
||||
);
|
||||
if (inlineMatch) return { reason: inlineMatch[1]!.trim() };
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
export class SceneExtractor {
|
||||
private dataDir: string;
|
||||
private runner: CleanContextRunner;
|
||||
private maxScenes: number;
|
||||
private sceneBackupCount: number;
|
||||
private timeoutMs: number;
|
||||
private logger: ExtractorLogger | undefined;
|
||||
private instanceId: string | undefined;
|
||||
|
||||
constructor(opts: SceneExtractorOptions) {
|
||||
this.dataDir = opts.dataDir;
|
||||
this.maxScenes = opts.maxScenes ?? 20;
|
||||
this.sceneBackupCount = opts.sceneBackupCount ?? 10;
|
||||
this.timeoutMs = opts.timeoutMs ?? 300_000; // 5 min — LLM may do multiple tool calls
|
||||
this.logger = opts.logger;
|
||||
this.instanceId = opts.instanceId;
|
||||
|
||||
this.runner = new CleanContextRunner({
|
||||
config: opts.config,
|
||||
modelRef: opts.model,
|
||||
enableTools: true,
|
||||
logger: opts.logger,
|
||||
});
|
||||
|
||||
this.logger?.debug?.(`${TAG} Created: dataDir=${opts.dataDir}, model=${opts.model ?? "(default)"}, maxScenes=${this.maxScenes}, timeout=${this.timeoutMs}ms`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract a batch of memories into scene blocks using the LLM agent.
|
||||
*
|
||||
* @param memories - Array of raw memory records from the API
|
||||
* @returns Extraction result with count and success flag
|
||||
*/
|
||||
async extract(memories: Array<{ content: string; created_at: string; id?: string }>): Promise<ExtractionResult> {
|
||||
const extractStartMs = Date.now();
|
||||
this.logger?.info(`${TAG} extract() start: ${memories.length} memories`);
|
||||
|
||||
if (memories.length === 0) {
|
||||
this.logger?.debug?.(`${TAG} extract() skipped: no memories`);
|
||||
return { memoriesProcessed: 0, success: true };
|
||||
}
|
||||
|
||||
const sceneBlocksDir = path.join(this.dataDir, "scene_blocks");
|
||||
const metadataDir = path.join(this.dataDir, ".metadata");
|
||||
|
||||
// Ensure directories exist
|
||||
await fs.mkdir(sceneBlocksDir, { recursive: true });
|
||||
await fs.mkdir(metadataDir, { recursive: true });
|
||||
|
||||
// Phase 1: Backup
|
||||
const backupStartMs = Date.now();
|
||||
const cpManager = new CheckpointManager(this.dataDir);
|
||||
const cp = await cpManager.read();
|
||||
const bm = new BackupManager(path.join(this.dataDir, ".backup"));
|
||||
await bm.backupDirectory(sceneBlocksDir, "scene_blocks", `offset${cp.total_processed}`, this.sceneBackupCount);
|
||||
this.logger?.debug?.(`${TAG} extract() backup phase: ${Date.now() - backupStartMs}ms`);
|
||||
|
||||
// Phase 2: Load scene index
|
||||
const indexStartMs = Date.now();
|
||||
const index = await readSceneIndex(this.dataDir);
|
||||
this.logger?.debug?.(`${TAG} extract() scene index loaded: ${index.length} entries (${Date.now() - indexStartMs}ms)`);
|
||||
|
||||
// Build scene summaries for the prompt (relative filenames only)
|
||||
const { summaries: sceneSummaries, filenames: existingSceneFiles } =
|
||||
this.buildSceneSummaries(index);
|
||||
|
||||
// Build scene count warning (tiered system)
|
||||
let sceneCountWarning: string | undefined;
|
||||
const sceneCount = index.length;
|
||||
if (sceneCount >= this.maxScenes) {
|
||||
sceneCountWarning = `当前场景数量为 **${sceneCount} 个**，已达到或超过 ${this.maxScenes} 个上限！\n**你必须先执行 MERGE 操作**，将最相似的 2-4 个场景合并为 1 个，然后再处理新记忆。\n参考合并对象：热度最低或主题高度重叠的场景。`;
|
||||
this.logger?.warn(`${TAG} extract() scene count at limit: ${sceneCount}/${this.maxScenes}`);
|
||||
} else if (sceneCount === this.maxScenes - 1) {
|
||||
sceneCountWarning = `当前场景数量为 **${sceneCount} 个**，距离上限只差 1 个！\n本次处理**只能 UPDATE 现有场景，不能 CREATE 新场景**。`;
|
||||
this.logger?.warn(`${TAG} extract() scene count near limit (CREATE blocked): ${sceneCount}/${this.maxScenes}`);
|
||||
} else if (sceneCount >= this.maxScenes - 3) {
|
||||
sceneCountWarning = `当前场景数量为 **${sceneCount} 个**，建议优先考虑 UPDATE 或主动 MERGE 相似场景。`;
|
||||
this.logger?.debug?.(`${TAG} extract() scene count approaching limit: ${sceneCount}/${this.maxScenes}`);
|
||||
}
|
||||
|
||||
// Snapshot scene index + content before LLM — used later to diff created/updated/deleted
|
||||
const preExtractIndex = new Map(index.map((e) => [e.filename, e.summary]));
|
||||
// Also snapshot scene content so we can detect content-only changes vs metadata-only changes
|
||||
const preExtractContent = new Map<string, string>();
|
||||
for (const e of index) {
|
||||
try {
|
||||
const raw = await fs.readFile(path.join(sceneBlocksDir, e.filename), "utf-8");
|
||||
const block = parseSceneBlock(raw, e.filename);
|
||||
preExtractContent.set(e.filename, block.content);
|
||||
} catch { /* non-fatal */ }
|
||||
}
|
||||
|
||||
// Phase 3: Build prompt
|
||||
const promptStartMs = Date.now();
|
||||
const memoriesJson = JSON.stringify(
|
||||
memories.map((m) => ({
|
||||
content: m.content,
|
||||
created_at: m.created_at,
|
||||
id: m.id ?? "",
|
||||
})),
|
||||
null,
|
||||
2,
|
||||
);
|
||||
|
||||
const currentTimestamp = formatTimestamp(new Date());
|
||||
|
||||
const prompt = buildSceneExtractionPrompt({
|
||||
memoriesJson,
|
||||
sceneSummaries: sceneSummaries || "(无已有场景)",
|
||||
currentTimestamp,
|
||||
sceneCountWarning,
|
||||
existingSceneFiles,
|
||||
});
|
||||
this.logger?.debug?.(`${TAG} extract() prompt built: ${prompt.length} chars (${Date.now() - promptStartMs}ms)`);
|
||||
|
||||
// Phase 4: Run LLM agent (sandboxed to scene_blocks/)
|
||||
let llmOutput = "";
|
||||
let llmDurationMs = 0;
|
||||
try {
|
||||
this.logger?.debug?.(`${TAG} extract() starting LLM runner (timeout=${this.timeoutMs}ms, maxTokens=model default)...`);
|
||||
const runnerStartMs = Date.now();
|
||||
llmOutput = await this.runner.run({
|
||||
prompt,
|
||||
taskId: `scene-extract-${Date.now()}`,
|
||||
timeoutMs: this.timeoutMs,
|
||||
// maxTokens omitted → core uses the resolved model's maxTokens from catalog
|
||||
workspaceDir: sceneBlocksDir,
|
||||
}) ?? "";
|
||||
llmDurationMs = Date.now() - runnerStartMs;
|
||||
this.logger?.debug?.(`${TAG} extract() LLM runner completed: ${llmDurationMs}ms`);
|
||||
} catch (err) {
|
||||
const errMsg = err instanceof Error ? err.message : String(err);
|
||||
const totalMs = Date.now() - extractStartMs;
|
||||
this.logger?.error(`${TAG} extract() LLM runner failed after ${totalMs}ms: ${errMsg}`);
|
||||
return { memoriesProcessed: 0, success: false, error: errMsg };
|
||||
}
|
||||
|
||||
// Phase 5: Subsequent processing — safe cleanup of soft-deleted files
|
||||
//
|
||||
// Security: The LLM has no `exec` tool and cannot run shell commands.
|
||||
// Instead, it "deletes" files by writing empty content (soft-delete).
|
||||
// Here we detect and remove those empty files before syncing the index,
|
||||
// so syncSceneIndex won't re-index stale empty entries.
|
||||
const cleanupStartMs = Date.now();
|
||||
let cleanedCount = 0;
|
||||
try {
|
||||
const allFiles = (await fs.readdir(sceneBlocksDir)).filter((f) => f.endsWith(".md"));
|
||||
for (const file of allFiles) {
|
||||
const filePath = path.join(sceneBlocksDir, file);
|
||||
const content = await fs.readFile(filePath, "utf-8");
|
||||
if (content.trim().length === 0) {
|
||||
await fs.unlink(filePath);
|
||||
cleanedCount++;
|
||||
this.logger?.debug?.(`${TAG} extract() removed soft-deleted file: ${file}`);
|
||||
}
|
||||
}
|
||||
} catch (cleanupErr) {
|
||||
// Non-fatal — log and continue to index sync
|
||||
this.logger?.warn(`${TAG} extract() soft-delete cleanup error: ${cleanupErr instanceof Error ? cleanupErr.message : String(cleanupErr)}`);
|
||||
}
|
||||
this.logger?.debug?.(`${TAG} extract() soft-delete cleanup: removed ${cleanedCount} empty files (${Date.now() - cleanupStartMs}ms)`);
|
||||
|
||||
// Phase 6: Sync scene index (rebuilds from remaining non-empty files)
|
||||
const syncStartMs = Date.now();
|
||||
await syncSceneIndex(this.dataDir);
|
||||
this.logger?.debug?.(`${TAG} extract() scene index synced: ${Date.now() - syncStartMs}ms`);
|
||||
|
||||
// Phase 7: Update persona.md navigation (GAP-4 fix)
|
||||
const navStartMs = Date.now();
|
||||
try {
|
||||
await this.updateSceneNavigation();
|
||||
this.logger?.debug?.(`${TAG} extract() persona.md navigation updated: ${Date.now() - navStartMs}ms`);
|
||||
} catch (navErr) {
|
||||
// Non-fatal — log and continue
|
||||
this.logger?.warn(`${TAG} extract() failed to update persona navigation: ${navErr instanceof Error ? navErr.message : String(navErr)}`);
|
||||
}
|
||||
|
||||
// Phase 8: Parse LLM output for out-of-band persona update signal
|
||||
if (llmOutput) {
|
||||
const signal = parsePersonaUpdateSignal(llmOutput);
|
||||
if (signal) {
|
||||
await cpManager.setPersonaUpdateRequest(signal.reason);
|
||||
this.logger?.debug?.(`${TAG} extract() persona update requested by LLM: ${signal.reason}`);
|
||||
}
|
||||
}
|
||||
|
||||
const totalMs = Date.now() - extractStartMs;
|
||||
this.logger?.info(`${TAG} extract() completed: ${memories.length} memories processed in ${totalMs}ms`);
|
||||
|
||||
// ── l2_extraction metric ──
|
||||
if (this.instanceId && this.logger) {
|
||||
// Read updated scene index to report final state + diff against pre-extract snapshot
|
||||
let resultScenes: Array<{ title: string; summary: string; content: string; status: "created" | "updated" }> = [];
|
||||
let scenesCreated = 0;
|
||||
let scenesUpdated = 0;
|
||||
let scenesDeleted = 0;
|
||||
try {
|
||||
const finalIndex = await readSceneIndex(this.dataDir);
|
||||
const postFilenames = new Set<string>();
|
||||
for (const e of finalIndex) {
|
||||
postFilenames.add(e.filename);
|
||||
const oldSummary = preExtractIndex.get(e.filename);
|
||||
// Read scene block content from disk
|
||||
let content = "";
|
||||
try {
|
||||
const blockPath = path.join(sceneBlocksDir, e.filename);
|
||||
const raw = await fs.readFile(blockPath, "utf-8");
|
||||
const block = parseSceneBlock(raw, e.filename);
|
||||
content = block.content;
|
||||
} catch { /* file read failure is non-fatal */ }
|
||||
|
||||
if (oldSummary === undefined) {
|
||||
// New scene
|
||||
scenesCreated++;
|
||||
resultScenes.push({
|
||||
title: e.filename.replace(/\.md$/, ""),
|
||||
summary: e.summary,
|
||||
content,
|
||||
status: "created",
|
||||
});
|
||||
} else {
|
||||
// Existing scene — check if content actually changed (not just metadata)
|
||||
const oldContent = preExtractContent.get(e.filename) ?? "";
|
||||
if (content !== oldContent) {
|
||||
scenesUpdated++;
|
||||
resultScenes.push({
|
||||
title: e.filename.replace(/\.md$/, ""),
|
||||
summary: e.summary,
|
||||
content,
|
||||
status: "updated",
|
||||
});
|
||||
}
|
||||
// If only metadata (summary/heat) changed but content is the same, skip
|
||||
}
|
||||
}
|
||||
// Scenes in pre-extract but missing from post-extract = deleted
|
||||
for (const [filename] of preExtractIndex) {
|
||||
if (!postFilenames.has(filename)) {
|
||||
scenesDeleted++;
|
||||
}
|
||||
}
|
||||
} catch { /* non-fatal */ }
|
||||
|
||||
report("l2_extraction", {
|
||||
inputMemoryCount: memories.length,
|
||||
resultSceneCount: resultScenes.length,
|
||||
resultScenes,
|
||||
scenesCreated,
|
||||
scenesUpdated,
|
||||
scenesDeleted,
|
||||
llmDurationMs,
|
||||
totalDurationMs: totalMs,
|
||||
success: true,
|
||||
error: null,
|
||||
});
|
||||
}
|
||||
|
||||
return { memoriesProcessed: memories.length, success: true };
|
||||
}
|
||||
|
||||
/**
|
||||
* Build human-readable scene summaries for the prompt,
|
||||
* and collect the list of existing scene filenames (relative).
|
||||
*
|
||||
* Includes a capacity counter at the top (e.g. "当前场景总数：5 / 15")
|
||||
* so the LLM can immediately see how close it is to the limit.
|
||||
*/
|
||||
private buildSceneSummaries(
|
||||
index: SceneIndexEntry[],
|
||||
): { summaries: string; filenames: string[] } {
|
||||
if (index.length === 0) return { summaries: "", filenames: [] };
|
||||
|
||||
const lines: string[] = [];
|
||||
const filenames: string[] = [];
|
||||
|
||||
// Inject capacity counter at the top — LLM sees this first
|
||||
lines.push(`**当前场景总数：${index.length} / ${this.maxScenes}**`);
|
||||
lines.push("");
|
||||
|
||||
for (const entry of index) {
|
||||
filenames.push(entry.filename);
|
||||
lines.push(`### ${entry.filename}`);
|
||||
lines.push(`**热度**: ${entry.heat} | **更新**: ${entry.updated}`);
|
||||
lines.push(`**summary**: ${entry.summary}`);
|
||||
lines.push("");
|
||||
}
|
||||
return { summaries: lines.join("\n"), filenames };
|
||||
}
|
||||
|
||||
/**
|
||||
* Update the scene navigation section at the end of persona.md.
|
||||
*
|
||||
* Reads the current scene index, generates the navigation block, then
|
||||
* strips any existing navigation from persona.md and appends the new one.
|
||||
*
|
||||
* IMPORTANT: If the persona body is empty (PersonaGenerator hasn't run yet),
|
||||
* we skip writing to avoid creating a persona.md that only contains the
|
||||
* scene navigation. PersonaGenerator.generate() will write the full
|
||||
* persona + navigation when it runs.
|
||||
*/
|
||||
private async updateSceneNavigation(): Promise<void> {
|
||||
const personaPath = path.join(this.dataDir, "persona.md");
|
||||
const index = await readSceneIndex(this.dataDir);
|
||||
const nav = generateSceneNavigation(index);
|
||||
|
||||
let existing = "";
|
||||
try {
|
||||
existing = await fs.readFile(personaPath, "utf-8");
|
||||
} catch {
|
||||
// No persona file yet — PersonaGenerator will create it with navigation.
|
||||
// Don't write a navigation-only file.
|
||||
this.logger?.debug?.(`${TAG} updateSceneNavigation() skipped: no persona file yet, waiting for PersonaGenerator`);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!existing.trim() && !nav) return;
|
||||
|
||||
const stripped = stripSceneNavigation(existing).trimEnd();
|
||||
|
||||
// If the persona body is empty (only navigation existed), don't overwrite
|
||||
// with a navigation-only file. Let PersonaGenerator handle full generation.
|
||||
if (!stripped) {
|
||||
this.logger?.debug?.(`${TAG} updateSceneNavigation() skipped: persona body is empty, waiting for PersonaGenerator`);
|
||||
return;
|
||||
}
|
||||
|
||||
const updated = nav ? `${stripped}\n\n${nav}\n` : `${stripped}\n`;
|
||||
|
||||
// persona.md is at dataDir root, no subdir needed
|
||||
await fs.writeFile(personaPath, updated, "utf-8");
|
||||
}
|
||||
}
|
||||
|
||||
function formatTimestamp(d: Date): string {
|
||||
return d.toISOString();
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
/**
|
||||
* Scene Block file format: parse and format the META-delimited Markdown files.
|
||||
*/
|
||||
|
||||
export interface SceneBlockMeta {
|
||||
created: string;
|
||||
updated: string;
|
||||
summary: string;
|
||||
heat: number;
|
||||
}
|
||||
|
||||
export interface SceneBlock {
|
||||
filename: string;
|
||||
meta: SceneBlockMeta;
|
||||
content: string;
|
||||
}
|
||||
|
||||
const META_START = "-----META-START-----";
|
||||
const META_END = "-----META-END-----";
|
||||
|
||||
/**
|
||||
* Parse a Scene Block file into structured data.
|
||||
*/
|
||||
export function parseSceneBlock(raw: string, filename: string): SceneBlock {
|
||||
const startIdx = raw.indexOf(META_START);
|
||||
const endIdx = raw.indexOf(META_END);
|
||||
|
||||
if (startIdx === -1 || endIdx === -1) {
|
||||
// No META section — treat entire file as content
|
||||
return {
|
||||
filename,
|
||||
meta: { created: "", updated: "", summary: "", heat: 0 },
|
||||
content: raw.trim(),
|
||||
};
|
||||
}
|
||||
|
||||
const metaBlock = raw.slice(startIdx + META_START.length, endIdx).trim();
|
||||
const content = raw.slice(endIdx + META_END.length).trim();
|
||||
|
||||
const meta: SceneBlockMeta = {
|
||||
created: extractMetaField(metaBlock, "created"),
|
||||
updated: extractMetaField(metaBlock, "updated"),
|
||||
summary: extractMetaField(metaBlock, "summary"),
|
||||
heat: parseInt(extractMetaField(metaBlock, "heat"), 10) || 0,
|
||||
};
|
||||
|
||||
return { filename, meta, content };
|
||||
}
|
||||
|
||||
/**
|
||||
* Format a Scene Block back into file content.
|
||||
*/
|
||||
export function formatSceneBlock(meta: SceneBlockMeta, content: string): string {
|
||||
return `${formatMeta(meta)}\n\n${content}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Format the META section.
|
||||
*/
|
||||
export function formatMeta(meta: SceneBlockMeta): string {
|
||||
return [
|
||||
META_START,
|
||||
`created: ${meta.created}`,
|
||||
`updated: ${meta.updated}`,
|
||||
`summary: ${meta.summary}`,
|
||||
`heat: ${meta.heat}`,
|
||||
META_END,
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function extractMetaField(metaBlock: string, field: string): string {
|
||||
const re = new RegExp(`^${field}:\\s*(.*)$`, "m");
|
||||
const m = metaBlock.match(re);
|
||||
return m ? m[1]!.trim() : "";
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
/**
|
||||
* Scene Index: maintains a JSON index of all scene blocks for quick lookup.
|
||||
*/
|
||||
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { parseSceneBlock } from "./scene-format.js";
|
||||
|
||||
export interface SceneIndexEntry {
|
||||
filename: string;
|
||||
summary: string;
|
||||
heat: number;
|
||||
created: string;
|
||||
updated: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the scene index from disk.
|
||||
*
|
||||
* The index is written exclusively by syncSceneIndex() (engineering side).
|
||||
* The LLM is sandboxed to scene_blocks/ and cannot access this file.
|
||||
*/
|
||||
export async function readSceneIndex(dataDir: string): Promise<SceneIndexEntry[]> {
|
||||
const indexPath = path.join(dataDir, ".metadata", "scene_index.json");
|
||||
try {
|
||||
const raw = await fs.readFile(indexPath, "utf-8");
|
||||
const parsed = JSON.parse(raw) as Array<Record<string, unknown>>;
|
||||
if (!Array.isArray(parsed)) return [];
|
||||
|
||||
const entries: SceneIndexEntry[] = [];
|
||||
for (const item of parsed) {
|
||||
if (!item || typeof item !== "object") continue;
|
||||
|
||||
const filename = typeof item.filename === "string" ? item.filename : "";
|
||||
if (!filename) continue;
|
||||
|
||||
entries.push({
|
||||
filename,
|
||||
summary: typeof item.summary === "string" ? item.summary : "",
|
||||
heat: typeof item.heat === "number" ? item.heat : 0,
|
||||
created: typeof item.created === "string" ? item.created : "",
|
||||
updated: typeof item.updated === "string" ? item.updated : "",
|
||||
});
|
||||
}
|
||||
return entries;
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Write the scene index to disk.
|
||||
*/
|
||||
export async function writeSceneIndex(
|
||||
dataDir: string,
|
||||
entries: SceneIndexEntry[],
|
||||
): Promise<void> {
|
||||
const indexPath = path.join(dataDir, ".metadata", "scene_index.json");
|
||||
await fs.mkdir(path.dirname(indexPath), { recursive: true });
|
||||
await fs.writeFile(indexPath, JSON.stringify(entries, null, 2), "utf-8");
|
||||
}
|
||||
|
||||
/**
|
||||
* Rebuild scene index by scanning all .md files in the scene_blocks directory.
|
||||
*/
|
||||
export async function syncSceneIndex(dataDir: string): Promise<SceneIndexEntry[]> {
|
||||
const blocksDir = path.join(dataDir, "scene_blocks");
|
||||
let files: string[];
|
||||
try {
|
||||
files = (await fs.readdir(blocksDir)).filter((f) => f.endsWith(".md"));
|
||||
} catch {
|
||||
files = [];
|
||||
}
|
||||
|
||||
const entries: SceneIndexEntry[] = [];
|
||||
for (const file of files) {
|
||||
try {
|
||||
const raw = await fs.readFile(path.join(blocksDir, file), "utf-8");
|
||||
const block = parseSceneBlock(raw, file);
|
||||
entries.push({
|
||||
filename: file,
|
||||
summary: block.meta.summary,
|
||||
heat: block.meta.heat,
|
||||
created: block.meta.created,
|
||||
updated: block.meta.updated,
|
||||
});
|
||||
} catch {
|
||||
// File may have been deleted between readdir and readFile (e.g. by concurrent
|
||||
// SceneExtractor soft-delete). Skip it and continue syncing the rest.
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
await writeSceneIndex(dataDir, entries);
|
||||
return entries;
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
/**
|
||||
* Scene navigation: generates a summary navigation section appended to persona.md.
|
||||
*
|
||||
* The navigation includes file paths so the agent can use read_file to load
|
||||
* scene details on demand (progressive disclosure).
|
||||
*/
|
||||
|
||||
import type { SceneIndexEntry } from "./scene-index.js";
|
||||
|
||||
const NAV_HEADER = "---\n## 🗺️ Scene Navigation (Scene Index)";
|
||||
|
||||
const NAV_FOOTER = `📌 使用说明：
|
||||
- Path 即 scene block 的相对路径，可按需使用 read_file 读取完整内容
|
||||
- 热度：该场景被记忆命中的累计次数，越高越重要
|
||||
- Summary：场景的核心要点摘要`;
|
||||
|
||||
/**
|
||||
* Build a fire-emoji string based on heat value (visual priority cue for the agent).
|
||||
*/
|
||||
function heatEmoji(heat: number): string {
|
||||
if (heat >= 1000) return " 🔥🔥🔥🔥🔥";
|
||||
if (heat >= 500) return " 🔥🔥🔥🔥";
|
||||
if (heat >= 200) return " 🔥🔥🔥";
|
||||
if (heat >= 100) return " 🔥🔥";
|
||||
if (heat >= 50) return " 🔥";
|
||||
return "";
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate the scene navigation Markdown section.
|
||||
*
|
||||
* Output format mirrors the v1 Python version so the agent can identify file
|
||||
* paths and invoke read_file for on-demand scene loading.
|
||||
*/
|
||||
export function generateSceneNavigation(entries: SceneIndexEntry[]): string {
|
||||
if (entries.length === 0) return "";
|
||||
|
||||
const sorted = [...entries].sort((a, b) => b.heat - a.heat);
|
||||
|
||||
const blocks = sorted.map((e) => {
|
||||
const pathLine = `### Path: scene_blocks/${e.filename}`;
|
||||
const heatLine = `**热度**: ${e.heat}${heatEmoji(e.heat)}${e.updated ? ` | **更新**: ${e.updated}` : ""}`;
|
||||
const summaryLine = `Summary: ${e.summary}`;
|
||||
return `${pathLine}\n${heatLine}\n${summaryLine}`;
|
||||
});
|
||||
|
||||
return `${NAV_HEADER}\n*以下是当前场景记忆的索引，可根据需要 read_file 读取详细内容。*\n\n${blocks.join("\n\n")}\n\n${NAV_FOOTER}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Strip the scene navigation section from persona content.
|
||||
*/
|
||||
export function stripSceneNavigation(personaContent: string): string {
|
||||
const idx = personaContent.indexOf(NAV_HEADER);
|
||||
if (idx === -1) return personaContent;
|
||||
return personaContent.slice(0, idx).trimEnd();
|
||||
}
|
||||
@@ -0,0 +1,598 @@
|
||||
/**
|
||||
* Embedding Service: converts text to vector embeddings.
|
||||
*
|
||||
* Supports two providers:
|
||||
* - "openai": OpenAI-compatible embedding APIs (OpenAI, Azure OpenAI, self-hosted)
|
||||
* - "local": node-llama-cpp with embeddinggemma-300m GGUF model (fully offline)
|
||||
*
|
||||
* When no remote embedding is configured, automatically falls back to local provider.
|
||||
*
|
||||
* Design:
|
||||
* - Single `embed()` for one text, `embedBatch()` for multiple.
|
||||
* - `getDimensions()` returns configured vector dimensions.
|
||||
* - Throws on failure; callers decide fallback strategy.
|
||||
*/
|
||||
|
||||
// ============================
|
||||
// Types
|
||||
// ============================
|
||||
|
||||
export interface OpenAIEmbeddingConfig {
|
||||
/** Provider identifier — any value other than "local" (e.g. "openai", "deepseek", "azure", "qclaw") */
|
||||
provider: string;
|
||||
/** API base URL (required — must be specified by user, e.g. "https://api.openai.com/v1") */
|
||||
baseUrl: string;
|
||||
/** API Key (required) */
|
||||
apiKey: string;
|
||||
/** Model name (required — must be specified by user) */
|
||||
model: string;
|
||||
/** Output dimensions (required — must match the chosen model) */
|
||||
dimensions: number;
|
||||
/** Local proxy URL (only for provider="qclaw") — requests are forwarded through this proxy with Remote-URL header */
|
||||
proxyUrl?: string;
|
||||
/** Max input text length in characters before truncation (default: 5000). */
|
||||
maxInputChars?: number;
|
||||
/** Timeout per API call in milliseconds (default: 10000). */
|
||||
timeoutMs?: number;
|
||||
}
|
||||
|
||||
export interface LocalEmbeddingConfig {
|
||||
provider: "local";
|
||||
/** Custom GGUF model path (default: embeddinggemma-300m from HuggingFace) */
|
||||
modelPath?: string;
|
||||
/** Model cache directory (default: node-llama-cpp default cache) */
|
||||
modelCacheDir?: string;
|
||||
}
|
||||
|
||||
export type EmbeddingConfig = OpenAIEmbeddingConfig | LocalEmbeddingConfig;
|
||||
|
||||
/** Identifies the embedding provider + model for change detection. */
|
||||
export interface EmbeddingProviderInfo {
|
||||
/** Provider identifier (e.g. "local", "openai", "deepseek") */
|
||||
provider: string;
|
||||
/** Model identifier (e.g. "embeddinggemma-300m", "text-embedding-3-large") */
|
||||
model: string;
|
||||
}
|
||||
|
||||
export interface EmbeddingService {
|
||||
/** Get embedding for a single text */
|
||||
embed(text: string): Promise<Float32Array>;
|
||||
/** Get embeddings for multiple texts (batched API call) */
|
||||
embedBatch(texts: string[]): Promise<Float32Array[]>;
|
||||
/** Return the configured vector dimensions */
|
||||
getDimensions(): number;
|
||||
/** Return provider + model identifiers for change detection */
|
||||
getProviderInfo(): EmbeddingProviderInfo;
|
||||
/**
|
||||
* Whether the service is ready to serve embed requests.
|
||||
* For remote providers (OpenAI), always true (stateless HTTP).
|
||||
* For local providers, true only after model download + load completes.
|
||||
*/
|
||||
isReady(): boolean;
|
||||
/**
|
||||
* Start background warmup (model download + load).
|
||||
* For remote providers, this is a no-op.
|
||||
* For local providers, triggers async initialization without blocking.
|
||||
* Safe to call multiple times (idempotent).
|
||||
*/
|
||||
startWarmup(): void;
|
||||
/** Optional: release resources (model memory, GPU, etc.) on shutdown */
|
||||
close?(): void | Promise<void>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Error thrown when embed() / embedBatch() is called before the local
|
||||
* embedding model has finished downloading and loading.
|
||||
* Callers should catch this and fall back to keyword-only mode.
|
||||
*/
|
||||
export class EmbeddingNotReadyError extends Error {
|
||||
constructor(message?: string) {
|
||||
super(message ?? "Local embedding model is not ready yet (still downloading or loading)");
|
||||
this.name = "EmbeddingNotReadyError";
|
||||
}
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Logger interface
|
||||
// ============================
|
||||
|
||||
interface Logger {
|
||||
debug?: (message: string) => void;
|
||||
info: (message: string) => void;
|
||||
warn: (message: string) => void;
|
||||
error: (message: string) => void;
|
||||
}
|
||||
|
||||
const TAG = "[memory-tdai][embedding]";
|
||||
|
||||
// ============================
|
||||
// Local (node-llama-cpp) implementation
|
||||
// ============================
|
||||
|
||||
/** Default model: Google's embeddinggemma-300m, quantized Q8_0 (~300MB) */
|
||||
const DEFAULT_LOCAL_MODEL =
|
||||
"hf:ggml-org/embeddinggemma-300m-qat-q8_0-GGUF/embeddinggemma-300m-qat-Q8_0.gguf";
|
||||
|
||||
/** embeddinggemma-300m outputs 768-dimensional vectors */
|
||||
const LOCAL_DIMENSIONS = 768;
|
||||
|
||||
/**
|
||||
* embeddinggemma-300m has a 256-token context window.
|
||||
* As a safe heuristic, we limit input to ~600 chars for CJK text
|
||||
* (CJK characters typically tokenize to 1-2 tokens each,
|
||||
* so 600 chars ≈ 200-400 tokens, keeping well within 256-token limit
|
||||
* after accounting for special tokens).
|
||||
* For Latin text, ~800 chars is a safe limit (~200 tokens).
|
||||
* We use 512 chars as a conservative universal limit.
|
||||
*/
|
||||
const LOCAL_MAX_INPUT_CHARS = 512;
|
||||
|
||||
/**
|
||||
* Sanitize NaN/Inf values and L2-normalize the vector.
|
||||
* Matches OpenClaw's own sanitizeAndNormalizeEmbedding().
|
||||
*/
|
||||
function sanitizeAndNormalize(vec: number[] | Float32Array): Float32Array {
|
||||
const arr = Array.from(vec).map((v) => (Number.isFinite(v) ? v : 0));
|
||||
const magnitude = Math.sqrt(arr.reduce((sum, v) => sum + v * v, 0));
|
||||
if (magnitude < 1e-10) {
|
||||
return new Float32Array(arr);
|
||||
}
|
||||
return new Float32Array(arr.map((v) => v / magnitude));
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialization state for LocalEmbeddingService.
|
||||
* - "idle": not started yet
|
||||
* - "initializing": model download / load is in progress (background)
|
||||
* - "ready": model is loaded and ready to serve
|
||||
* - "failed": initialization failed (will retry on next startWarmup)
|
||||
*/
|
||||
type LocalInitState = "idle" | "initializing" | "ready" | "failed";
|
||||
|
||||
export class LocalEmbeddingService implements EmbeddingService {
|
||||
private readonly modelPath: string;
|
||||
private readonly modelCacheDir?: string;
|
||||
private readonly logger?: Logger;
|
||||
|
||||
// Initialization state machine
|
||||
private initState: LocalInitState = "idle";
|
||||
private initPromise: Promise<void> | null = null;
|
||||
private initError: Error | null = null;
|
||||
private embeddingContext: {
|
||||
getEmbeddingFor: (text: string) => Promise<{ vector: Float32Array | number[] }>;
|
||||
} | null = null;
|
||||
|
||||
constructor(config?: LocalEmbeddingConfig, logger?: Logger) {
|
||||
this.modelPath = config?.modelPath?.trim() || DEFAULT_LOCAL_MODEL;
|
||||
this.modelCacheDir = config?.modelCacheDir?.trim();
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
getDimensions(): number {
|
||||
return LOCAL_DIMENSIONS;
|
||||
}
|
||||
|
||||
getProviderInfo(): EmbeddingProviderInfo {
|
||||
return { provider: "local", model: this.modelPath };
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether the local model is fully loaded and ready to serve requests.
|
||||
*/
|
||||
isReady(): boolean {
|
||||
return this.initState === "ready" && this.embeddingContext !== null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Start background warmup: download model (if needed) and load into memory.
|
||||
* Does NOT block the caller — returns immediately.
|
||||
* Safe to call multiple times (idempotent); re-triggers on "failed" state.
|
||||
*/
|
||||
startWarmup(): void {
|
||||
if (this.initState === "initializing" || this.initState === "ready") {
|
||||
return; // already in progress or done
|
||||
}
|
||||
this.logger?.info(`${TAG} Starting background warmup for local embedding model...`);
|
||||
this.initState = "initializing";
|
||||
this.initError = null;
|
||||
|
||||
this.initPromise = this._doInitialize()
|
||||
.then(() => {
|
||||
this.initState = "ready";
|
||||
this.logger?.info(`${TAG} Background warmup complete — local embedding ready`);
|
||||
})
|
||||
.catch((err) => {
|
||||
this.initState = "failed";
|
||||
this.initError = err instanceof Error ? err : new Error(String(err));
|
||||
this.logger?.error(
|
||||
`${TAG} Background warmup failed: ${this.initError.message}. ` +
|
||||
`embed() calls will throw EmbeddingNotReadyError until retried.`,
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Get embedding for a single text.
|
||||
* @throws {EmbeddingNotReadyError} if model is not yet ready.
|
||||
*/
|
||||
async embed(text: string): Promise<Float32Array> {
|
||||
this.assertReady();
|
||||
const truncated = this.truncateInput(text);
|
||||
const embedding = await this.embeddingContext!.getEmbeddingFor(truncated);
|
||||
return sanitizeAndNormalize(embedding.vector);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get embeddings for multiple texts.
|
||||
* @throws {EmbeddingNotReadyError} if model is not yet ready.
|
||||
*/
|
||||
async embedBatch(texts: string[]): Promise<Float32Array[]> {
|
||||
if (texts.length === 0) return [];
|
||||
this.assertReady();
|
||||
|
||||
const results: Float32Array[] = [];
|
||||
for (const text of texts) {
|
||||
const truncated = this.truncateInput(text);
|
||||
const embedding = await this.embeddingContext!.getEmbeddingFor(truncated);
|
||||
results.push(sanitizeAndNormalize(embedding.vector));
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Release the node-llama-cpp embedding context and model resources.
|
||||
* Safe to call multiple times (idempotent).
|
||||
*/
|
||||
close(): void {
|
||||
if (this.embeddingContext) {
|
||||
try {
|
||||
const ctx = this.embeddingContext as unknown as { dispose?: () => void };
|
||||
ctx.dispose?.();
|
||||
} catch {
|
||||
// best-effort cleanup
|
||||
}
|
||||
this.embeddingContext = null;
|
||||
this.initPromise = null;
|
||||
this.initState = "idle";
|
||||
this.initError = null;
|
||||
this.logger?.info(`${TAG} Local embedding resources released`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Assert the model is ready. Throws EmbeddingNotReadyError if not.
|
||||
*/
|
||||
private assertReady(): void {
|
||||
if (this.initState === "ready" && this.embeddingContext) {
|
||||
return;
|
||||
}
|
||||
if (this.initState === "failed") {
|
||||
throw new EmbeddingNotReadyError(
|
||||
`Local embedding model initialization failed: ${this.initError?.message ?? "unknown error"}. ` +
|
||||
`Call startWarmup() to retry.`,
|
||||
);
|
||||
}
|
||||
if (this.initState === "initializing") {
|
||||
throw new EmbeddingNotReadyError(
|
||||
"Local embedding model is still loading (download/initialization in progress). Please try again later.",
|
||||
);
|
||||
}
|
||||
// "idle" — startWarmup() was never called
|
||||
throw new EmbeddingNotReadyError(
|
||||
"Local embedding model warmup has not been started. Call startWarmup() first.",
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Truncate input text to stay within the model's context window.
|
||||
* embeddinggemma-300m has a 256-token limit; we use a character-based
|
||||
* heuristic (LOCAL_MAX_INPUT_CHARS) as a safe proxy.
|
||||
*/
|
||||
private truncateInput(text: string): string {
|
||||
if (text.length <= LOCAL_MAX_INPUT_CHARS) return text;
|
||||
this.logger?.debug?.(
|
||||
`${TAG} Input truncated from ${text.length} to ${LOCAL_MAX_INPUT_CHARS} chars (model context limit)`,
|
||||
);
|
||||
return text.slice(0, LOCAL_MAX_INPUT_CHARS);
|
||||
}
|
||||
|
||||
/**
|
||||
* Internal: perform the actual model download + load.
|
||||
* Called by startWarmup(), runs in background.
|
||||
*/
|
||||
private async _doInitialize(): Promise<void> {
|
||||
// Track partially-initialized resources for cleanup on failure
|
||||
let model: { createEmbeddingContext: () => Promise<unknown>; dispose?: () => void } | undefined;
|
||||
try {
|
||||
this.logger?.debug?.(`${TAG} Loading node-llama-cpp for local embedding...`);
|
||||
|
||||
// Dynamic import — node-llama-cpp is a peer dependency of OpenClaw
|
||||
const { getLlama, resolveModelFile, LlamaLogLevel } = await import("node-llama-cpp");
|
||||
|
||||
const llama = await getLlama({ logLevel: LlamaLogLevel.error });
|
||||
this.logger?.debug?.(`${TAG} Llama instance created`);
|
||||
|
||||
const resolvedPath = await resolveModelFile(
|
||||
this.modelPath,
|
||||
this.modelCacheDir || undefined,
|
||||
);
|
||||
this.logger?.debug?.(`${TAG} Model resolved: ${resolvedPath}`);
|
||||
|
||||
model = await (llama as unknown as { loadModel: (opts: { modelPath: string }) => Promise<typeof model> }).loadModel({ modelPath: resolvedPath });
|
||||
this.logger?.debug?.(`${TAG} Model loaded, creating embedding context...`);
|
||||
|
||||
this.embeddingContext = await model!.createEmbeddingContext() as typeof this.embeddingContext;
|
||||
this.logger?.info(`${TAG} Local embedding ready (model=${this.modelPath}, dims=${LOCAL_DIMENSIONS})`);
|
||||
} catch (err) {
|
||||
// Clean up partially-initialized resources to prevent leaks
|
||||
if (model?.dispose) {
|
||||
try { model.dispose(); } catch { /* best-effort */ }
|
||||
}
|
||||
this.embeddingContext = null;
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait for ongoing warmup to complete (used internally by tests).
|
||||
* Returns immediately if already ready or idle.
|
||||
*/
|
||||
async waitForReady(): Promise<void> {
|
||||
if (this.initPromise) {
|
||||
await this.initPromise;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ============================
|
||||
// OpenAI-compatible implementation
|
||||
// ============================
|
||||
|
||||
/** Max texts per batch (OpenAI limit is 2048, we use a safe value) */
|
||||
const MAX_BATCH_SIZE = 256;
|
||||
|
||||
/** Max retries for API calls */
|
||||
const MAX_RETRIES = 0;
|
||||
/** Default timeout per API call in milliseconds */
|
||||
const DEFAULT_API_TIMEOUT_MS = 10_000;
|
||||
|
||||
/**
|
||||
* Custom error class for embedding API errors that carries HTTP status code.
|
||||
* Used to distinguish non-retryable client errors (4xx except 429) from
|
||||
* retryable server errors (5xx) and rate limits (429).
|
||||
*/
|
||||
class EmbeddingApiError extends Error {
|
||||
readonly httpStatus: number;
|
||||
constructor(message: string, httpStatus: number) {
|
||||
super(message);
|
||||
this.name = "EmbeddingApiError";
|
||||
this.httpStatus = httpStatus;
|
||||
}
|
||||
/** Returns true for 4xx errors that should NOT be retried (excluding 429). */
|
||||
isClientError(): boolean {
|
||||
return this.httpStatus >= 400 && this.httpStatus < 500 && this.httpStatus !== 429;
|
||||
}
|
||||
}
|
||||
|
||||
interface OpenAIEmbeddingResponse {
|
||||
data: Array<{
|
||||
index: number;
|
||||
embedding: number[];
|
||||
}>;
|
||||
usage?: {
|
||||
prompt_tokens: number;
|
||||
total_tokens: number;
|
||||
};
|
||||
}
|
||||
|
||||
export class OpenAIEmbeddingService implements EmbeddingService {
|
||||
private readonly baseUrl: string;
|
||||
private readonly apiKey: string;
|
||||
private readonly model: string;
|
||||
private readonly dims: number;
|
||||
private readonly providerName: string;
|
||||
private readonly proxyUrl?: string;
|
||||
private readonly maxInputChars?: number;
|
||||
private readonly timeoutMs: number;
|
||||
private readonly logger?: Logger;
|
||||
|
||||
constructor(config: OpenAIEmbeddingConfig, logger?: Logger) {
|
||||
if (!config.apiKey) {
|
||||
throw new Error("EmbeddingService: apiKey is required for remote provider");
|
||||
}
|
||||
if (!config.baseUrl) {
|
||||
throw new Error("EmbeddingService: baseUrl is required for remote provider");
|
||||
}
|
||||
if (!config.model) {
|
||||
throw new Error("EmbeddingService: model is required for remote provider");
|
||||
}
|
||||
if (!config.dimensions || config.dimensions <= 0) {
|
||||
throw new Error("EmbeddingService: dimensions is required for remote provider (must be a positive integer)");
|
||||
}
|
||||
this.baseUrl = config.baseUrl.replace(/\/+$/, "");
|
||||
this.apiKey = config.apiKey;
|
||||
this.model = config.model;
|
||||
this.dims = config.dimensions;
|
||||
this.providerName = config.provider || "openai";
|
||||
this.proxyUrl = config.proxyUrl?.trim() || undefined;
|
||||
this.maxInputChars = config.maxInputChars && config.maxInputChars > 0 ? config.maxInputChars : undefined;
|
||||
this.timeoutMs = config.timeoutMs && config.timeoutMs > 0 ? config.timeoutMs : DEFAULT_API_TIMEOUT_MS;
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
getDimensions(): number {
|
||||
return this.dims;
|
||||
}
|
||||
|
||||
getProviderInfo(): EmbeddingProviderInfo {
|
||||
return { provider: this.providerName, model: this.model };
|
||||
}
|
||||
|
||||
/** Remote embedding is always ready (stateless HTTP). */
|
||||
isReady(): boolean {
|
||||
return true;
|
||||
}
|
||||
|
||||
/** No-op for remote embedding (no local model to warm up). */
|
||||
startWarmup(): void {
|
||||
// nothing to do — remote API is stateless
|
||||
}
|
||||
|
||||
async embed(text: string): Promise<Float32Array> {
|
||||
const [result] = await this.embedBatch([text]);
|
||||
return result;
|
||||
}
|
||||
|
||||
async embedBatch(texts: string[]): Promise<Float32Array[]> {
|
||||
if (texts.length === 0) return [];
|
||||
|
||||
// Truncate texts exceeding maxInputChars limit
|
||||
const processedTexts = this.maxInputChars
|
||||
? texts.map((t) => this.truncateInput(t))
|
||||
: texts;
|
||||
|
||||
// Split into sub-batches if needed
|
||||
if (processedTexts.length > MAX_BATCH_SIZE) {
|
||||
const results: Float32Array[] = [];
|
||||
for (let i = 0; i < processedTexts.length; i += MAX_BATCH_SIZE) {
|
||||
const chunk = processedTexts.slice(i, i + MAX_BATCH_SIZE);
|
||||
const chunkResults = await this._callApi(chunk);
|
||||
results.push(...chunkResults);
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
return this._callApi(processedTexts);
|
||||
}
|
||||
|
||||
/**
|
||||
* Truncate input text to stay within the configured maxInputChars limit.
|
||||
* Logs a warning when truncation occurs.
|
||||
*/
|
||||
private truncateInput(text: string): string {
|
||||
if (!this.maxInputChars || text.length <= this.maxInputChars) return text;
|
||||
this.logger?.warn?.(
|
||||
`${TAG} Input truncated from ${text.length} to ${this.maxInputChars} chars (maxInputChars limit)`,
|
||||
);
|
||||
return text.slice(0, this.maxInputChars);
|
||||
}
|
||||
|
||||
private async _callApi(texts: string[]): Promise<Float32Array[]> {
|
||||
const body: Record<string, unknown> = {
|
||||
input: texts,
|
||||
model: this.model,
|
||||
dimensions: this.dims,
|
||||
};
|
||||
|
||||
// Determine fetch URL and headers based on proxy mode
|
||||
const useProxy = this.providerName === "qclaw" && !!this.proxyUrl;
|
||||
const fetchUrl = useProxy ? this.proxyUrl! : `${this.baseUrl}/embeddings`;
|
||||
const headers: Record<string, string> = {
|
||||
"Content-Type": "application/json",
|
||||
Authorization: `Bearer ${this.apiKey}`,
|
||||
};
|
||||
if (useProxy) {
|
||||
headers["Remote-URL"] = `${this.baseUrl}/embeddings`;
|
||||
this.logger?.debug?.(
|
||||
`${TAG} [qclaw-proxy] Forwarding embedding request via proxy: ${fetchUrl}, Remote-URL: ${headers["Remote-URL"]}`,
|
||||
);
|
||||
}
|
||||
|
||||
// Retry loop with timeout
|
||||
let lastError: Error | undefined;
|
||||
for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
|
||||
try {
|
||||
const controller = new AbortController();
|
||||
const timeoutId = setTimeout(() => controller.abort(), this.timeoutMs);
|
||||
|
||||
try {
|
||||
const resp = await fetch(fetchUrl, {
|
||||
method: "POST",
|
||||
headers,
|
||||
body: JSON.stringify(body),
|
||||
signal: controller.signal,
|
||||
});
|
||||
|
||||
if (!resp.ok) {
|
||||
const errBody = await resp.text().catch(() => "(unable to read body)");
|
||||
const err = new EmbeddingApiError(
|
||||
`Embedding API error: HTTP ${resp.status} ${resp.statusText} — ${errBody.slice(0, 500)}`,
|
||||
resp.status,
|
||||
);
|
||||
// Don't retry on 4xx client errors (except 429 rate limit)
|
||||
if (resp.status >= 400 && resp.status < 500 && resp.status !== 429) {
|
||||
throw err;
|
||||
}
|
||||
lastError = err;
|
||||
continue;
|
||||
}
|
||||
|
||||
const json = (await resp.json()) as OpenAIEmbeddingResponse;
|
||||
|
||||
if (!json.data || !Array.isArray(json.data)) {
|
||||
throw new Error("Embedding API returned unexpected format: missing 'data' array");
|
||||
}
|
||||
|
||||
// Sort by index to ensure correct order, then sanitize+normalize for consistency with local provider
|
||||
const sorted = [...json.data].sort((a, b) => a.index - b.index);
|
||||
return sorted.map((d) => sanitizeAndNormalize(d.embedding));
|
||||
} finally {
|
||||
clearTimeout(timeoutId);
|
||||
}
|
||||
} catch (err) {
|
||||
// Non-retryable errors (4xx client errors) — rethrow immediately
|
||||
if (err instanceof EmbeddingApiError && err.isClientError()) {
|
||||
throw err;
|
||||
}
|
||||
lastError = err instanceof Error ? err : new Error(String(err));
|
||||
// AbortError = timeout, retry
|
||||
if (attempt < MAX_RETRIES) {
|
||||
// Exponential backoff: 500ms, 1000ms
|
||||
const delay = 500 * (attempt + 1);
|
||||
await new Promise((r) => setTimeout(r, delay));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
throw lastError ?? new Error("Embedding API call failed after retries");
|
||||
}
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Factory
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Create an EmbeddingService from config.
|
||||
*
|
||||
* Strategy:
|
||||
* - If config has provider != "local" with valid apiKey, model, and dimensions → use remote OpenAI-compatible embedding
|
||||
* - If config has provider="local" → use node-llama-cpp local embedding
|
||||
* - If config is undefined or missing required fields → fall back to local embedding
|
||||
*
|
||||
* NOTE: For local providers, `startWarmup()` is NOT called here.
|
||||
* The caller is responsible for calling `startWarmup()` at the right time
|
||||
* (e.g. on first conversation) to avoid triggering model download during
|
||||
* short-lived CLI commands like `gateway stop` or `agents list`.
|
||||
*/
|
||||
export function createEmbeddingService(
|
||||
config: EmbeddingConfig | undefined,
|
||||
logger?: Logger,
|
||||
): EmbeddingService {
|
||||
// Remote OpenAI-compatible provider: any provider value other than "local"
|
||||
if (config && config.provider !== "local" && "apiKey" in config && config.apiKey) {
|
||||
logger?.info(`${TAG} Using remote embedding (provider=${config.provider}, model=${config.model})`);
|
||||
return new OpenAIEmbeddingService(config as OpenAIEmbeddingConfig, logger);
|
||||
}
|
||||
|
||||
// Explicit local config
|
||||
if (config && config.provider === "local") {
|
||||
const localConfig = config as LocalEmbeddingConfig;
|
||||
logger?.info(`${TAG} Using local embedding (node-llama-cpp, model=${localConfig.modelPath ?? DEFAULT_LOCAL_MODEL})`);
|
||||
return new LocalEmbeddingService(localConfig, logger);
|
||||
}
|
||||
|
||||
// Fallback: no config or empty apiKey → use local
|
||||
logger?.info(`${TAG} No remote embedding configured, falling back to local embedding (node-llama-cpp)`);
|
||||
return new LocalEmbeddingService(undefined, logger);
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,279 @@
|
||||
/**
|
||||
* conversation_search tool: Agent-callable tool for searching L0 conversation records.
|
||||
*
|
||||
* Supports three search strategies with automatic degradation:
|
||||
* 1. **hybrid** (default) — FTS5 keyword + vector embedding in parallel,
|
||||
* merged via Reciprocal Rank Fusion (RRF).
|
||||
* 2. **embedding** — pure vector similarity (when FTS5 is unavailable).
|
||||
* 3. **fts** — pure FTS5 keyword search (when embedding is unavailable).
|
||||
*
|
||||
* The tool is registered via `api.registerTool()` in index.ts.
|
||||
*/
|
||||
|
||||
import type { VectorStore, L0VectorSearchResult } from "../store/vector-store.js";
|
||||
import { buildFtsQuery } from "../store/vector-store.js";
|
||||
import type { EmbeddingService } from "../store/embedding.js";
|
||||
|
||||
// ============================
|
||||
// Types
|
||||
// ============================
|
||||
|
||||
interface Logger {
|
||||
debug?: (message: string) => void;
|
||||
info: (message: string) => void;
|
||||
warn: (message: string) => void;
|
||||
error: (message: string) => void;
|
||||
}
|
||||
|
||||
export interface ConversationSearchResultItem {
|
||||
id: string;
|
||||
session_key: string;
|
||||
/** Role of the message sender: "user" or "assistant" */
|
||||
role: string;
|
||||
/** Text content of this single message */
|
||||
content: string;
|
||||
score: number;
|
||||
recorded_at: string;
|
||||
}
|
||||
|
||||
export interface ConversationSearchResult {
|
||||
results: ConversationSearchResultItem[];
|
||||
total: number;
|
||||
/** Actual search strategy used: "hybrid", "embedding", "fts", or "none". */
|
||||
strategy: string;
|
||||
/** Optional message, e.g. when embedding is not configured. */
|
||||
message?: string;
|
||||
}
|
||||
|
||||
const TAG = "[memory-tdai][tdai_conversation_search]";
|
||||
|
||||
// ============================
|
||||
// RRF (Reciprocal Rank Fusion)
|
||||
// ============================
|
||||
|
||||
/** Standard RRF constant from the original RRF paper. */
|
||||
const RRF_K = 60;
|
||||
|
||||
/**
|
||||
* Merge multiple ranked lists of `ConversationSearchResultItem` via Reciprocal
|
||||
* Rank Fusion. Items appearing in multiple lists get their RRF scores summed.
|
||||
*
|
||||
* Returns items sorted by descending RRF score. The `score` field of each
|
||||
* returned item is replaced by the RRF score for consistent ranking semantics.
|
||||
*/
|
||||
function rrfMergeL0(...lists: ConversationSearchResultItem[][]): ConversationSearchResultItem[] {
|
||||
const map = new Map<string, { item: ConversationSearchResultItem; rrfScore: number }>();
|
||||
|
||||
for (const list of lists) {
|
||||
for (let rank = 0; rank < list.length; rank++) {
|
||||
const item = list[rank];
|
||||
const score = 1 / (RRF_K + rank + 1);
|
||||
const existing = map.get(item.id);
|
||||
if (existing) {
|
||||
existing.rrfScore += score;
|
||||
} else {
|
||||
map.set(item.id, { item, rrfScore: score });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return [...map.values()]
|
||||
.sort((a, b) => b.rrfScore - a.rrfScore)
|
||||
.map(({ item, rrfScore }) => ({ ...item, score: rrfScore }));
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Search implementation
|
||||
// ============================
|
||||
|
||||
export async function executeConversationSearch(params: {
|
||||
query: string;
|
||||
limit: number;
|
||||
sessionKey?: string;
|
||||
vectorStore?: VectorStore;
|
||||
embeddingService?: EmbeddingService;
|
||||
logger?: Logger;
|
||||
}): Promise<ConversationSearchResult> {
|
||||
const {
|
||||
query,
|
||||
limit,
|
||||
sessionKey: sessionFilter,
|
||||
vectorStore,
|
||||
embeddingService,
|
||||
logger,
|
||||
} = params;
|
||||
|
||||
logger?.debug?.(
|
||||
`${TAG} CALLED: query="${query.slice(0, 100)}", limit=${limit}, ` +
|
||||
`sessionFilter=${sessionFilter ?? "(none)"}, ` +
|
||||
`vectorStore=${vectorStore ? "available" : "UNAVAILABLE"}, ` +
|
||||
`embeddingService=${embeddingService ? "available" : "UNAVAILABLE"}`,
|
||||
);
|
||||
|
||||
if (!query || query.trim().length === 0) {
|
||||
logger?.debug?.(`${TAG} Empty query, returning empty`);
|
||||
return { results: [], total: 0, strategy: "none" };
|
||||
}
|
||||
|
||||
if (!vectorStore) {
|
||||
logger?.warn?.(`${TAG} VectorStore not available`);
|
||||
return { results: [], total: 0, strategy: "none" };
|
||||
}
|
||||
|
||||
// ── Determine available capabilities ──
|
||||
const hasEmbedding = !!embeddingService;
|
||||
const hasFts = vectorStore.isFtsAvailable();
|
||||
|
||||
if (!hasEmbedding && !hasFts) {
|
||||
logger?.warn?.(`${TAG} Neither EmbeddingService nor FTS5 available — cannot search`);
|
||||
return {
|
||||
results: [],
|
||||
total: 0,
|
||||
strategy: "none",
|
||||
message:
|
||||
"Embedding service is not configured and FTS is not available. " +
|
||||
"Conversation search requires an embedding provider or FTS5 support. " +
|
||||
"Please configure an embedding provider in the embedding.provider setting (e.g. openai_compatible).",
|
||||
};
|
||||
}
|
||||
|
||||
// ── Over-retrieve for later filtering and RRF merging ──
|
||||
const candidateK = sessionFilter ? limit * 4 : limit * 3;
|
||||
|
||||
// ── Run available search strategies in parallel ──
|
||||
const [ftsItems, vecItems] = await Promise.all([
|
||||
// FTS5 keyword search on L0
|
||||
(async (): Promise<ConversationSearchResultItem[]> => {
|
||||
if (!hasFts) return [];
|
||||
try {
|
||||
const ftsQuery = buildFtsQuery(query);
|
||||
if (!ftsQuery) {
|
||||
logger?.debug?.(`${TAG} [hybrid-fts] No usable FTS tokens from query`);
|
||||
return [];
|
||||
}
|
||||
logger?.debug?.(`${TAG} [hybrid-fts] FTS5 query: "${ftsQuery}"`);
|
||||
const ftsResults = vectorStore.ftsSearchL0(ftsQuery, candidateK);
|
||||
logger?.debug?.(`${TAG} [hybrid-fts] FTS5 returned ${ftsResults.length} candidates`);
|
||||
return ftsResults.map((r) => ({
|
||||
id: r.record_id,
|
||||
session_key: r.session_key,
|
||||
role: r.role,
|
||||
content: r.message_text,
|
||||
score: r.score,
|
||||
recorded_at: r.recorded_at,
|
||||
}));
|
||||
} catch (err) {
|
||||
logger?.warn?.(
|
||||
`${TAG} [hybrid-fts] FTS5 search failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
return [];
|
||||
}
|
||||
})(),
|
||||
|
||||
// Vector embedding search on L0
|
||||
(async (): Promise<ConversationSearchResultItem[]> => {
|
||||
if (!hasEmbedding) return [];
|
||||
try {
|
||||
logger?.debug?.(`${TAG} [hybrid-vec] Generating query embedding...`);
|
||||
const queryEmbedding = await embeddingService!.embed(query);
|
||||
logger?.debug?.(
|
||||
`${TAG} [hybrid-vec] Embedding OK, dims=${queryEmbedding.length}, searching top-${candidateK}...`,
|
||||
);
|
||||
const vecResults: L0VectorSearchResult[] = vectorStore.searchL0(queryEmbedding, candidateK);
|
||||
logger?.debug?.(`${TAG} [hybrid-vec] Vector search returned ${vecResults.length} candidates`);
|
||||
return vecResults.map((r) => ({
|
||||
id: r.record_id,
|
||||
session_key: r.session_key,
|
||||
role: r.role,
|
||||
content: r.message_text,
|
||||
score: r.score,
|
||||
recorded_at: r.recorded_at,
|
||||
}));
|
||||
} catch (err) {
|
||||
logger?.warn?.(
|
||||
`${TAG} [hybrid-vec] Embedding search failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
return [];
|
||||
}
|
||||
})(),
|
||||
]);
|
||||
|
||||
// ── Determine effective strategy ──
|
||||
const ftsOk = ftsItems.length > 0;
|
||||
const vecOk = vecItems.length > 0;
|
||||
let strategy: string;
|
||||
|
||||
if (ftsOk && vecOk) {
|
||||
strategy = "hybrid";
|
||||
} else if (vecOk) {
|
||||
strategy = "embedding";
|
||||
} else if (ftsOk) {
|
||||
strategy = "fts";
|
||||
} else {
|
||||
logger?.debug?.(`${TAG} Both search paths returned 0 results`);
|
||||
return { results: [], total: 0, strategy: hasEmbedding ? "embedding" : "fts" };
|
||||
}
|
||||
|
||||
// ── Merge results ──
|
||||
let results: ConversationSearchResultItem[];
|
||||
if (strategy === "hybrid") {
|
||||
results = rrfMergeL0(ftsItems, vecItems);
|
||||
logger?.debug?.(
|
||||
`${TAG} [hybrid] RRF merged: fts=${ftsItems.length}, vec=${vecItems.length} → ${results.length} unique`,
|
||||
);
|
||||
} else {
|
||||
// Single-source: use whichever list has results (already sorted by score)
|
||||
results = ftsOk ? ftsItems : vecItems;
|
||||
}
|
||||
|
||||
// ── Apply session key filter ──
|
||||
if (sessionFilter) {
|
||||
const preFilterCount = results.length;
|
||||
results = results.filter((r) => r.session_key === sessionFilter);
|
||||
logger?.debug?.(`${TAG} After session filter "${sessionFilter}": ${results.length}/${preFilterCount}`);
|
||||
}
|
||||
|
||||
// ── Trim to requested limit ──
|
||||
const trimmed = results.slice(0, limit);
|
||||
|
||||
logger?.debug?.(
|
||||
`${TAG} RESULT (strategy=${strategy}): returning ${trimmed.length} messages ` +
|
||||
`(scores: [${trimmed.map((r) => r.score.toFixed(3)).join(", ")}])`,
|
||||
);
|
||||
|
||||
return {
|
||||
results: trimmed,
|
||||
total: trimmed.length,
|
||||
strategy,
|
||||
};
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Tool response formatter
|
||||
// ============================
|
||||
|
||||
export function formatConversationSearchResponse(result: ConversationSearchResult): string {
|
||||
if (result.message) {
|
||||
return result.message;
|
||||
}
|
||||
if (result.results.length === 0) {
|
||||
return "No matching conversation messages found.";
|
||||
}
|
||||
|
||||
const lines: string[] = [
|
||||
`Found ${result.total} matching message(s):`,
|
||||
"",
|
||||
];
|
||||
|
||||
for (const item of result.results) {
|
||||
const scoreStr = typeof item.score === "number" ? ` (score: ${item.score.toFixed(3)})` : "";
|
||||
const dateStr = item.recorded_at ? ` [${item.recorded_at}]` : "";
|
||||
lines.push(`---`);
|
||||
lines.push(`**[${item.role}]** Session: ${item.session_key}${dateStr}${scoreStr}`);
|
||||
lines.push("");
|
||||
lines.push(item.content);
|
||||
lines.push("");
|
||||
}
|
||||
|
||||
return lines.join("\n");
|
||||
}
|
||||
@@ -0,0 +1,290 @@
|
||||
/**
|
||||
* memory_search tool: Agent-callable tool for searching L1 memory records.
|
||||
*
|
||||
* Supports three search strategies with automatic degradation:
|
||||
* 1. **hybrid** (default) — FTS5 keyword + vector embedding in parallel,
|
||||
* merged via Reciprocal Rank Fusion (RRF).
|
||||
* 2. **embedding** — pure vector similarity (when FTS5 is unavailable).
|
||||
* 3. **fts** — pure FTS5 keyword search (when embedding is unavailable).
|
||||
*
|
||||
* The tool is registered via `api.registerTool()` in index.ts.
|
||||
*/
|
||||
|
||||
import type { VectorStore, VectorSearchResult } from "../store/vector-store.js";
|
||||
import { buildFtsQuery } from "../store/vector-store.js";
|
||||
import type { EmbeddingService } from "../store/embedding.js";
|
||||
|
||||
// ============================
|
||||
// Types
|
||||
// ============================
|
||||
|
||||
interface Logger {
|
||||
debug?: (message: string) => void;
|
||||
info: (message: string) => void;
|
||||
warn: (message: string) => void;
|
||||
error: (message: string) => void;
|
||||
}
|
||||
|
||||
export interface MemorySearchResultItem {
|
||||
id: string;
|
||||
content: string;
|
||||
type: string;
|
||||
priority: number;
|
||||
scene_name: string;
|
||||
score: number;
|
||||
created_at: string;
|
||||
updated_at: string;
|
||||
}
|
||||
|
||||
export interface MemorySearchResult {
|
||||
results: MemorySearchResultItem[];
|
||||
total: number;
|
||||
strategy: string;
|
||||
/** Optional message, e.g. when embedding is not configured. */
|
||||
message?: string;
|
||||
}
|
||||
|
||||
const TAG = "[memory-tdai][tdai_memory_search]";
|
||||
|
||||
// ============================
|
||||
// RRF (Reciprocal Rank Fusion)
|
||||
// ============================
|
||||
|
||||
/** Standard RRF constant from the original RRF paper. */
|
||||
const RRF_K = 60;
|
||||
|
||||
/**
|
||||
* Merge multiple ranked lists of `MemorySearchResultItem` via Reciprocal Rank
|
||||
* Fusion. Items appearing in multiple lists get their RRF scores summed.
|
||||
*
|
||||
* Returns items sorted by descending RRF score. The `score` field of each
|
||||
* returned item is replaced by the RRF score for consistent ranking semantics.
|
||||
*/
|
||||
function rrfMergeL1(...lists: MemorySearchResultItem[][]): MemorySearchResultItem[] {
|
||||
const map = new Map<string, { item: MemorySearchResultItem; rrfScore: number }>();
|
||||
|
||||
for (const list of lists) {
|
||||
for (let rank = 0; rank < list.length; rank++) {
|
||||
const item = list[rank];
|
||||
const score = 1 / (RRF_K + rank + 1);
|
||||
const existing = map.get(item.id);
|
||||
if (existing) {
|
||||
existing.rrfScore += score;
|
||||
} else {
|
||||
map.set(item.id, { item, rrfScore: score });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return [...map.values()]
|
||||
.sort((a, b) => b.rrfScore - a.rrfScore)
|
||||
.map(({ item, rrfScore }) => ({ ...item, score: rrfScore }));
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Search implementation
|
||||
// ============================
|
||||
|
||||
export async function executeMemorySearch(params: {
|
||||
query: string;
|
||||
limit: number;
|
||||
type?: string;
|
||||
scene?: string;
|
||||
vectorStore?: VectorStore;
|
||||
embeddingService?: EmbeddingService;
|
||||
logger?: Logger;
|
||||
}): Promise<MemorySearchResult> {
|
||||
const {
|
||||
query,
|
||||
limit,
|
||||
type: typeFilter,
|
||||
scene: sceneFilter,
|
||||
vectorStore,
|
||||
embeddingService,
|
||||
logger,
|
||||
} = params;
|
||||
|
||||
logger?.debug?.(
|
||||
`${TAG} CALLED: query="${query.slice(0, 100)}", limit=${limit}, ` +
|
||||
`typeFilter=${typeFilter ?? "(none)"}, sceneFilter=${sceneFilter ?? "(none)"}, ` +
|
||||
`vectorStore=${vectorStore ? "available" : "UNAVAILABLE"}, ` +
|
||||
`embeddingService=${embeddingService ? "available" : "UNAVAILABLE"}`,
|
||||
);
|
||||
|
||||
if (!query || query.trim().length === 0) {
|
||||
logger?.debug?.(`${TAG} Empty query, returning empty`);
|
||||
return { results: [], total: 0, strategy: "none" };
|
||||
}
|
||||
|
||||
if (!vectorStore) {
|
||||
logger?.warn?.(`${TAG} VectorStore not available`);
|
||||
return { results: [], total: 0, strategy: "none" };
|
||||
}
|
||||
|
||||
// ── Determine available capabilities ──
|
||||
const hasEmbedding = !!embeddingService;
|
||||
const hasFts = vectorStore.isFtsAvailable();
|
||||
|
||||
if (!hasEmbedding && !hasFts) {
|
||||
logger?.warn?.(`${TAG} Neither EmbeddingService nor FTS5 available — cannot search`);
|
||||
return {
|
||||
results: [],
|
||||
total: 0,
|
||||
strategy: "none",
|
||||
message:
|
||||
"Embedding service is not configured and FTS is not available. " +
|
||||
"Memory search requires an embedding provider or FTS5 support. " +
|
||||
"Please configure an embedding provider in the embedding.provider setting (e.g. openai_compatible).",
|
||||
};
|
||||
}
|
||||
|
||||
// ── Over-retrieve for later filtering and RRF merging ──
|
||||
const candidateK = limit * 3;
|
||||
|
||||
// ── Run available search strategies in parallel ──
|
||||
const [ftsItems, vecItems] = await Promise.all([
|
||||
// FTS5 keyword search
|
||||
(async (): Promise<MemorySearchResultItem[]> => {
|
||||
if (!hasFts) return [];
|
||||
try {
|
||||
const ftsQuery = buildFtsQuery(query);
|
||||
if (!ftsQuery) {
|
||||
logger?.debug?.(`${TAG} [hybrid-fts] No usable FTS tokens from query`);
|
||||
return [];
|
||||
}
|
||||
logger?.debug?.(`${TAG} [hybrid-fts] FTS5 query: "${ftsQuery}"`);
|
||||
const ftsResults = vectorStore.ftsSearchL1(ftsQuery, candidateK);
|
||||
logger?.debug?.(`${TAG} [hybrid-fts] FTS5 returned ${ftsResults.length} candidates`);
|
||||
return ftsResults.map((r) => ({
|
||||
id: r.record_id,
|
||||
content: r.content,
|
||||
type: r.type,
|
||||
priority: r.priority,
|
||||
scene_name: r.scene_name,
|
||||
score: r.score,
|
||||
created_at: r.timestamp_start,
|
||||
updated_at: r.timestamp_end,
|
||||
}));
|
||||
} catch (err) {
|
||||
logger?.warn?.(
|
||||
`${TAG} [hybrid-fts] FTS5 search failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
return [];
|
||||
}
|
||||
})(),
|
||||
|
||||
// Vector embedding search
|
||||
(async (): Promise<MemorySearchResultItem[]> => {
|
||||
if (!hasEmbedding) return [];
|
||||
try {
|
||||
logger?.debug?.(`${TAG} [hybrid-vec] Generating query embedding...`);
|
||||
const queryEmbedding = await embeddingService!.embed(query);
|
||||
logger?.debug?.(
|
||||
`${TAG} [hybrid-vec] Embedding OK, dims=${queryEmbedding.length}, searching top-${candidateK}...`,
|
||||
);
|
||||
const vecResults: VectorSearchResult[] = vectorStore.search(queryEmbedding, candidateK);
|
||||
logger?.debug?.(`${TAG} [hybrid-vec] Vector search returned ${vecResults.length} candidates`);
|
||||
return vecResults.map((r) => ({
|
||||
id: r.record_id,
|
||||
content: r.content,
|
||||
type: r.type,
|
||||
priority: r.priority,
|
||||
scene_name: r.scene_name,
|
||||
score: r.score,
|
||||
created_at: r.timestamp_start,
|
||||
updated_at: r.timestamp_end,
|
||||
}));
|
||||
} catch (err) {
|
||||
logger?.warn?.(
|
||||
`${TAG} [hybrid-vec] Embedding search failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
return [];
|
||||
}
|
||||
})(),
|
||||
]);
|
||||
|
||||
// ── Determine effective strategy ──
|
||||
const ftsOk = ftsItems.length > 0;
|
||||
const vecOk = vecItems.length > 0;
|
||||
let strategy: string;
|
||||
|
||||
if (ftsOk && vecOk) {
|
||||
strategy = "hybrid";
|
||||
} else if (vecOk) {
|
||||
strategy = "embedding";
|
||||
} else if (ftsOk) {
|
||||
strategy = "fts";
|
||||
} else {
|
||||
logger?.debug?.(`${TAG} Both search paths returned 0 results`);
|
||||
return { results: [], total: 0, strategy: hasEmbedding ? "embedding" : "fts" };
|
||||
}
|
||||
|
||||
// ── Merge results ──
|
||||
let results: MemorySearchResultItem[];
|
||||
if (strategy === "hybrid") {
|
||||
results = rrfMergeL1(ftsItems, vecItems);
|
||||
logger?.debug?.(
|
||||
`${TAG} [hybrid] RRF merged: fts=${ftsItems.length}, vec=${vecItems.length} → ${results.length} unique`,
|
||||
);
|
||||
} else {
|
||||
// Single-source: use whichever list has results (already sorted by score)
|
||||
results = ftsOk ? ftsItems : vecItems;
|
||||
}
|
||||
|
||||
// ── Apply secondary filters (type, scene) ──
|
||||
const preFilterCount = results.length;
|
||||
if (typeFilter) {
|
||||
results = results.filter((r) => r.type === typeFilter);
|
||||
logger?.debug?.(`${TAG} After type filter "${typeFilter}": ${results.length}/${preFilterCount}`);
|
||||
}
|
||||
if (sceneFilter) {
|
||||
const normalizedScene = sceneFilter.toLowerCase();
|
||||
results = results.filter((r) =>
|
||||
r.scene_name.toLowerCase().includes(normalizedScene),
|
||||
);
|
||||
logger?.debug?.(`${TAG} After scene filter "${sceneFilter}": ${results.length}/${preFilterCount}`);
|
||||
}
|
||||
|
||||
// ── Trim to requested limit ──
|
||||
const trimmed = results.slice(0, limit);
|
||||
|
||||
logger?.debug?.(
|
||||
`${TAG} RESULT (strategy=${strategy}): returning ${trimmed.length} memories ` +
|
||||
`(scores: [${trimmed.map((r) => r.score.toFixed(3)).join(", ")}])`,
|
||||
);
|
||||
|
||||
return {
|
||||
results: trimmed,
|
||||
total: trimmed.length,
|
||||
strategy,
|
||||
};
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Tool response formatter
|
||||
// ============================
|
||||
|
||||
export function formatSearchResponse(result: MemorySearchResult): string {
|
||||
if (result.message) {
|
||||
return result.message;
|
||||
}
|
||||
if (result.results.length === 0) {
|
||||
return "No matching memories found.";
|
||||
}
|
||||
|
||||
const lines: string[] = [
|
||||
`Found ${result.total} matching memories:`,
|
||||
"",
|
||||
];
|
||||
|
||||
for (const item of result.results) {
|
||||
const scoreStr = typeof item.score === "number" ? ` (score: ${item.score.toFixed(3)})` : "";
|
||||
const sceneStr = item.scene_name ? ` [scene: ${item.scene_name}]` : "";
|
||||
const priorityStr = item.priority >= 0 ? ` (priority: ${item.priority})` : " (global instruction)";
|
||||
lines.push(`- **[${item.type}]**${priorityStr}${sceneStr}${scoreStr}`);
|
||||
lines.push(` ${item.content}`);
|
||||
lines.push("");
|
||||
}
|
||||
|
||||
return lines.join("\n");
|
||||
}
|
||||
@@ -0,0 +1,156 @@
|
||||
/**
|
||||
* BackupManager: generic file/directory backup utility.
|
||||
*
|
||||
* Provides two backup modes:
|
||||
* - `backupFile(src, category, tag, maxKeep)` — copy a single file
|
||||
* - `backupDirectory(src, category, tag, maxKeep)` — copy an entire directory
|
||||
*
|
||||
* All backups land under `<backupRoot>/<category>/` with timestamped names.
|
||||
* After each backup, entries beyond `maxKeep` are automatically pruned
|
||||
* (oldest first, by lexicographic order on the timestamp-embedded name).
|
||||
*/
|
||||
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
|
||||
export class BackupManager {
|
||||
private backupRoot: string;
|
||||
|
||||
/**
|
||||
* @param backupRoot - Absolute path to the root backup directory
|
||||
* (e.g. `<dataDir>/.backup`).
|
||||
*/
|
||||
constructor(backupRoot: string) {
|
||||
this.backupRoot = backupRoot;
|
||||
}
|
||||
|
||||
/**
|
||||
* Backup a single file.
|
||||
*
|
||||
* Destination: `<backupRoot>/<category>/<category>_<timestamp>_<tag>.<ext>`
|
||||
*
|
||||
* @param srcFile - Absolute path to the source file
|
||||
* @param category - Logical grouping (e.g. "persona")
|
||||
* @param tag - Additional identifier (e.g. "offset42")
|
||||
* @param maxKeep - Max backup files to retain in this category (0 = unlimited)
|
||||
*/
|
||||
async backupFile(
|
||||
srcFile: string,
|
||||
category: string,
|
||||
tag: string,
|
||||
maxKeep: number,
|
||||
): Promise<void> {
|
||||
try {
|
||||
await fs.access(srcFile);
|
||||
} catch {
|
||||
return; // Source file doesn't exist, nothing to backup
|
||||
}
|
||||
|
||||
const destDir = path.join(this.backupRoot, category);
|
||||
await fs.mkdir(destDir, { recursive: true });
|
||||
|
||||
const ext = path.extname(srcFile); // e.g. ".md"
|
||||
const timestamp = formatTimestamp(new Date());
|
||||
const destName = `${category}_${timestamp}_${tag}${ext}`;
|
||||
await fs.copyFile(srcFile, path.join(destDir, destName));
|
||||
|
||||
if (maxKeep > 0) {
|
||||
await pruneOldEntries(destDir, maxKeep, "file");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Backup an entire directory (shallow copy of all files).
|
||||
*
|
||||
* Destination: `<backupRoot>/<category>/<category>_<timestamp>_<tag>/`
|
||||
*
|
||||
* @param srcDir - Absolute path to the source directory
|
||||
* @param category - Logical grouping (e.g. "scene_blocks")
|
||||
* @param tag - Additional identifier (e.g. "offset42")
|
||||
* @param maxKeep - Max backup directories to retain in this category (0 = unlimited)
|
||||
*/
|
||||
async backupDirectory(
|
||||
srcDir: string,
|
||||
category: string,
|
||||
tag: string,
|
||||
maxKeep: number,
|
||||
): Promise<void> {
|
||||
let entries: import("node:fs").Dirent[];
|
||||
try {
|
||||
entries = await fs.readdir(srcDir, { withFileTypes: true });
|
||||
} catch {
|
||||
return; // Source directory doesn't exist
|
||||
}
|
||||
|
||||
// Only backup regular files (skip subdirectories to avoid EISDIR errors)
|
||||
const files = entries.filter((e) => e.isFile()).map((e) => e.name);
|
||||
if (files.length === 0) return;
|
||||
|
||||
const parentDir = path.join(this.backupRoot, category);
|
||||
const timestamp = formatTimestamp(new Date());
|
||||
const destDir = path.join(parentDir, `${category}_${timestamp}_${tag}`);
|
||||
await fs.mkdir(destDir, { recursive: true });
|
||||
|
||||
for (const file of files) {
|
||||
await fs.copyFile(path.join(srcDir, file), path.join(destDir, file));
|
||||
}
|
||||
|
||||
if (maxKeep > 0) {
|
||||
await pruneOldEntries(parentDir, maxKeep, "directory");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Helpers
|
||||
// ============================
|
||||
|
||||
function formatTimestamp(d: Date): string {
|
||||
const pad = (n: number) => String(n).padStart(2, "0");
|
||||
return [
|
||||
d.getFullYear(),
|
||||
pad(d.getMonth() + 1),
|
||||
pad(d.getDate()),
|
||||
"_",
|
||||
pad(d.getHours()),
|
||||
pad(d.getMinutes()),
|
||||
pad(d.getSeconds()),
|
||||
].join("");
|
||||
}
|
||||
|
||||
/**
|
||||
* Keep only the newest `maxKeep` entries in a directory.
|
||||
* Entries are sorted by name ascending (oldest first) since backup names
|
||||
* embed timestamps, so lexicographic order = chronological order.
|
||||
*
|
||||
* @param dir - Directory containing the backup entries
|
||||
* @param maxKeep - Number of entries to retain
|
||||
* @param kind - "file" to unlink, "directory" to rm -rf
|
||||
*/
|
||||
async function pruneOldEntries(
|
||||
dir: string,
|
||||
maxKeep: number,
|
||||
kind: "file" | "directory",
|
||||
): Promise<void> {
|
||||
let entries: string[];
|
||||
try {
|
||||
entries = await fs.readdir(dir);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
|
||||
entries.sort(); // ascending — oldest first
|
||||
const toRemove = entries.slice(0, Math.max(0, entries.length - maxKeep));
|
||||
|
||||
for (const name of toRemove) {
|
||||
try {
|
||||
if (kind === "file") {
|
||||
await fs.unlink(path.join(dir, name));
|
||||
} else {
|
||||
await fs.rm(path.join(dir, name), { recursive: true, force: true });
|
||||
}
|
||||
} catch {
|
||||
// best-effort
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,543 @@
|
||||
/**
|
||||
* Checkpoint management for tracking memory processing progress.
|
||||
*
|
||||
* ## Split-state design
|
||||
*
|
||||
* Per-session state is split into two independent namespaces to prevent
|
||||
* the PipelineManager and L0/L1 runners from overwriting each other's fields:
|
||||
*
|
||||
* - **runner_states** (`RunnerSessionState`): owned by CheckpointManager methods
|
||||
* (markL1*, advanceSession*). Contains L0 capture cursor, L1 cursor, scene name.
|
||||
*
|
||||
* - **pipeline_states** (`PipelineSessionState`): owned exclusively by
|
||||
* PipelineManager via `mergePipelineStates()`. Contains conversation_count,
|
||||
* extraction times, L2 tracking fields.
|
||||
*
|
||||
* Each side only reads/writes its own namespace, eliminating the split-brain
|
||||
* overwrite bug where pipeline persistStates() could clobber runner-written fields.
|
||||
*
|
||||
* ## Concurrency safety
|
||||
*
|
||||
* All mutating methods (read-modify-write) are serialized via a per-file async lock.
|
||||
* Multiple CheckpointManager instances sharing the same file path automatically share
|
||||
* the same lock, so callers can freely `new CheckpointManager()` without coordination.
|
||||
* Writes use atomic tmp+rename to prevent corruption on crash.
|
||||
*/
|
||||
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { randomBytes } from "node:crypto";
|
||||
|
||||
// ============================
|
||||
// Types
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Per-session state managed by L0/L1 runners (written directly to checkpoint).
|
||||
* These fields are ONLY written by CheckpointManager methods (markL1*, advanceSession*, etc.)
|
||||
* and are NEVER touched by the PipelineManager's persistStates().
|
||||
*/
|
||||
export interface RunnerSessionState {
|
||||
// ═══ L0 — per-session capture cursor ═══
|
||||
/** Epoch ms of the newest message captured for THIS session.
|
||||
* Used instead of the global `Checkpoint.last_captured_timestamp` so that
|
||||
* concurrent sessions don't advance each other's cursors and cause missed messages. */
|
||||
last_captured_timestamp: number;
|
||||
|
||||
// ═══ L1 — cursor & continuity ═══
|
||||
/** L0 JSONL cursor: epoch ms of last message processed by L1 */
|
||||
last_l1_cursor: number;
|
||||
/** Last scene name from the most recent L1 extraction (for cross-batch continuity) */
|
||||
last_scene_name: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Per-session state managed exclusively by PipelineManager (written via mergePipelineStates).
|
||||
* These fields are ONLY written by the pipeline's persistStates() callback
|
||||
* and are NEVER touched by CheckpointManager's L0/L1 methods.
|
||||
*/
|
||||
export interface PipelineSessionState {
|
||||
/** Conversation rounds since last L1 trigger */
|
||||
conversation_count: number;
|
||||
/** ISO timestamp of the last extraction completion */
|
||||
last_extraction_time: string;
|
||||
/** ISO timestamp cursor for incremental extraction reads */
|
||||
last_extraction_updated_time: string;
|
||||
/** Epoch ms of the last notifyConversation call */
|
||||
last_active_time: number;
|
||||
/** Mirrors conversation_count at L1 completion time (for L2 tracking) */
|
||||
l2_pending_l1_count: number;
|
||||
/**
|
||||
* Current warm-up threshold for L1 triggering.
|
||||
* Starts at 1 for new sessions and doubles after each L1 completion
|
||||
* (1 → 2 → 4 → 8 → ...) until it reaches everyNConversations.
|
||||
* 0 means warm-up is complete (use everyNConversations directly).
|
||||
*/
|
||||
warmup_threshold: number;
|
||||
/** ISO timestamp of last L2 extraction completion */
|
||||
l2_last_extraction_time: string;
|
||||
}
|
||||
|
||||
export interface Checkpoint {
|
||||
// ═══ Global counters ═══
|
||||
/** Epoch ms of the newest message successfully uploaded. Messages with ts > this are new. */
|
||||
last_captured_timestamp: number;
|
||||
/** Total messages processed across all time */
|
||||
total_processed: number;
|
||||
last_persona_at: number;
|
||||
last_persona_time: string;
|
||||
request_persona_update: boolean;
|
||||
persona_update_reason: string;
|
||||
memories_since_last_persona: number;
|
||||
scenes_processed: number;
|
||||
|
||||
// ═══ Per-session split state ═══
|
||||
/** Runner-managed per-session state (L0 capture cursor, L1 cursor, scene name).
|
||||
* Written ONLY by CheckpointManager methods. */
|
||||
runner_states: Record<string, RunnerSessionState>;
|
||||
/** Pipeline-managed per-session state (conversation_count, extraction times, etc.).
|
||||
* Written ONLY by the pipeline's mergePipelineStates(). */
|
||||
pipeline_states: Record<string, PipelineSessionState>;
|
||||
|
||||
// ═══ L0 ═══
|
||||
/** Total L0 conversation files recorded */
|
||||
l0_conversations_count: number;
|
||||
|
||||
// ═══ L1 ═══
|
||||
/** Total L1 memories extracted across all time */
|
||||
total_memories_extracted: number;
|
||||
}
|
||||
|
||||
const DEFAULT_RUNNER_STATE: RunnerSessionState = {
|
||||
last_captured_timestamp: 0,
|
||||
last_l1_cursor: 0,
|
||||
last_scene_name: "",
|
||||
};
|
||||
|
||||
const DEFAULT_PIPELINE_STATE: PipelineSessionState = {
|
||||
conversation_count: 0,
|
||||
last_extraction_time: "",
|
||||
last_extraction_updated_time: "",
|
||||
last_active_time: 0,
|
||||
l2_pending_l1_count: 0,
|
||||
warmup_threshold: 0, // 0 = graduated (safe default for old sessions missing this field)
|
||||
l2_last_extraction_time: "",
|
||||
};
|
||||
|
||||
const DEFAULT_CHECKPOINT: Checkpoint = {
|
||||
last_captured_timestamp: 0,
|
||||
total_processed: 0,
|
||||
last_persona_at: 0,
|
||||
last_persona_time: "",
|
||||
request_persona_update: false,
|
||||
persona_update_reason: "",
|
||||
memories_since_last_persona: 0,
|
||||
scenes_processed: 0,
|
||||
runner_states: {},
|
||||
pipeline_states: {},
|
||||
l0_conversations_count: 0,
|
||||
total_memories_extracted: 0,
|
||||
};
|
||||
|
||||
export interface CheckpointLogger {
|
||||
info(msg: string): void;
|
||||
warn?(msg: string): void;
|
||||
}
|
||||
|
||||
const noopLogger: CheckpointLogger = { info() {} };
|
||||
|
||||
// ============================
|
||||
// Per-file async lock
|
||||
// ============================
|
||||
// Keyed by resolved file path. Multiple CheckpointManager instances pointing
|
||||
// to the same file automatically share the same lock — callers don't need to
|
||||
// coordinate instance creation.
|
||||
|
||||
const fileLocks = new Map<string, Promise<void>>();
|
||||
|
||||
/**
|
||||
* Serialize async critical sections per file path.
|
||||
* Under no contention the overhead is a single resolved-promise await.
|
||||
*/
|
||||
async function withFileLock<T>(filePath: string, fn: () => Promise<T>): Promise<T> {
|
||||
// Chain after whatever is currently queued for this path
|
||||
const prev = fileLocks.get(filePath) ?? Promise.resolve();
|
||||
let release!: () => void;
|
||||
const gate = new Promise<void>((r) => { release = r; });
|
||||
fileLocks.set(filePath, gate);
|
||||
|
||||
await prev;
|
||||
try {
|
||||
return await fn();
|
||||
} finally {
|
||||
release();
|
||||
// Clean up the map entry if we're the tail of the chain
|
||||
if (fileLocks.get(filePath) === gate) {
|
||||
fileLocks.delete(filePath);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export class CheckpointManager {
|
||||
private filePath: string;
|
||||
private logger: CheckpointLogger;
|
||||
|
||||
constructor(dataDir: string, logger?: CheckpointLogger) {
|
||||
this.filePath = path.join(dataDir, ".metadata", "recall_checkpoint.json");
|
||||
this.logger = logger ?? noopLogger;
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Low-level I/O (internal)
|
||||
// ============================
|
||||
|
||||
private async readRaw(): Promise<Checkpoint> {
|
||||
try {
|
||||
const raw = await fs.readFile(this.filePath, "utf-8");
|
||||
const parsed = JSON.parse(raw) as Record<string, unknown>;
|
||||
// Merge with defaults for backward compat (old checkpoints lack new fields).
|
||||
// structuredClone avoids shallow-copy pitfall: without it, the nested
|
||||
// runner_states/pipeline_states objects in DEFAULT_CHECKPOINT would be
|
||||
// shared across all callers and mutated in place — corrupting the default.
|
||||
const cp = { ...structuredClone(DEFAULT_CHECKPOINT), ...parsed } as Checkpoint;
|
||||
|
||||
// Migrate from old session_states format (pre-split)
|
||||
const oldStates = parsed.session_states as Record<string, Record<string, unknown>> | undefined;
|
||||
if (oldStates && !parsed.runner_states && !parsed.pipeline_states) {
|
||||
cp.runner_states = {};
|
||||
cp.pipeline_states = {};
|
||||
for (const [key, state] of Object.entries(oldStates)) {
|
||||
cp.runner_states[key] = {
|
||||
...DEFAULT_RUNNER_STATE,
|
||||
last_captured_timestamp: (state.last_captured_timestamp as number) ?? 0,
|
||||
last_l1_cursor: (state.last_l1_cursor as number) ?? 0,
|
||||
last_scene_name: (state.last_scene_name as string) ?? "",
|
||||
};
|
||||
cp.pipeline_states[key] = {
|
||||
...DEFAULT_PIPELINE_STATE,
|
||||
conversation_count: (state.conversation_count as number) ?? 0,
|
||||
last_extraction_time: (state.last_extraction_time as string) ?? "",
|
||||
last_extraction_updated_time: (state.last_extraction_updated_time as string) ?? "",
|
||||
last_active_time: (state.last_active_time as number) ?? 0,
|
||||
l2_pending_l1_count: (state.l2_pending_l1_count as number) ?? 0,
|
||||
l2_last_extraction_time: (state.l2_last_extraction_time as string) ?? "",
|
||||
};
|
||||
}
|
||||
} else {
|
||||
// Ensure per-session states have all fields with defaults
|
||||
if (cp.runner_states) {
|
||||
for (const [key, state] of Object.entries(cp.runner_states)) {
|
||||
cp.runner_states[key] = { ...DEFAULT_RUNNER_STATE, ...state };
|
||||
}
|
||||
}
|
||||
if (cp.pipeline_states) {
|
||||
for (const [key, state] of Object.entries(cp.pipeline_states)) {
|
||||
cp.pipeline_states[key] = { ...DEFAULT_PIPELINE_STATE, ...state };
|
||||
}
|
||||
}
|
||||
}
|
||||
return cp;
|
||||
} catch {
|
||||
return structuredClone(DEFAULT_CHECKPOINT);
|
||||
}
|
||||
}
|
||||
|
||||
/** Atomic write: write to tmp file, then rename into place. */
|
||||
private async writeRaw(checkpoint: Checkpoint): Promise<void> {
|
||||
const dir = path.dirname(this.filePath);
|
||||
await fs.mkdir(dir, { recursive: true });
|
||||
const tmp = `${this.filePath}.tmp.${randomBytes(4).toString("hex")}`;
|
||||
await fs.writeFile(tmp, JSON.stringify(checkpoint, null, 2), "utf-8");
|
||||
await fs.rename(tmp, this.filePath);
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Locked read-modify-write helper
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Execute a mutating operation under the per-file lock.
|
||||
* `fn` receives the current checkpoint and may modify it in place;
|
||||
* the updated checkpoint is atomically written back.
|
||||
*/
|
||||
private async mutate(fn: (cp: Checkpoint) => void | Promise<void>): Promise<Checkpoint> {
|
||||
return withFileLock(this.filePath, async () => {
|
||||
const cp = await this.readRaw();
|
||||
await fn(cp);
|
||||
await this.writeRaw(cp);
|
||||
return cp;
|
||||
});
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Public API — read-only
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Read the current checkpoint (unlocked snapshot).
|
||||
*
|
||||
* NOTE: This does NOT acquire the file lock. The returned snapshot may be
|
||||
* stale if a concurrent `mutate()` is in progress. This is acceptable for
|
||||
* read-only uses (status display, deciding whether to run a pipeline step).
|
||||
*
|
||||
* For read-then-write patterns, always use `mutate()` instead — it acquires
|
||||
* the lock and re-reads from disk inside the critical section, ensuring the
|
||||
* update is based on the latest state.
|
||||
*/
|
||||
async read(): Promise<Checkpoint> {
|
||||
return this.readRaw();
|
||||
}
|
||||
|
||||
/** Write a full checkpoint (acquires lock + atomic write). */
|
||||
async write(checkpoint: Checkpoint): Promise<void> {
|
||||
return withFileLock(this.filePath, () => this.writeRaw(checkpoint));
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Public API — mutating (all serialized via file lock)
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Advance the captured timestamp after successful upload/recording.
|
||||
* Also updates total_processed and persona counters.
|
||||
*
|
||||
* NOTE: This advances the GLOBAL cursor (`Checkpoint.last_captured_timestamp`).
|
||||
* For per-session cursor advancement, use `advanceSessionCapturedTimestamp()`.
|
||||
* The global cursor is kept for aggregate stats / backward compat, but should
|
||||
* NOT be used as the L0 incremental-capture filter (use per-session instead).
|
||||
*/
|
||||
async advanceCapturedTimestamp(maxTimestamp: number, messageCount: number): Promise<void> {
|
||||
const cp = await this.mutate((cp) => {
|
||||
cp.last_captured_timestamp = maxTimestamp;
|
||||
cp.total_processed += messageCount;
|
||||
cp.memories_since_last_persona += messageCount;
|
||||
});
|
||||
this.logger.info(
|
||||
`[checkpoint] advanceCapturedTimestamp: -> ${maxTimestamp} (+${messageCount} msgs), ` +
|
||||
`total_processed=${cp.total_processed}, memories_since_last_persona=${cp.memories_since_last_persona}`,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Advance the per-session L0 capture cursor after recording messages.
|
||||
* This is the **primary** cursor for incremental L0 recording — each session
|
||||
* tracks its own progress independently, preventing cross-session cursor drift.
|
||||
*
|
||||
* Also updates the global cursor / total_processed for aggregate stats.
|
||||
*/
|
||||
async advanceSessionCapturedTimestamp(
|
||||
sessionKey: string,
|
||||
maxTimestamp: number,
|
||||
messageCount: number,
|
||||
): Promise<void> {
|
||||
const cp = await this.mutate((cp) => {
|
||||
// Per-session cursor (runner-owned)
|
||||
const state = this.getRunnerState(cp, sessionKey);
|
||||
state.last_captured_timestamp = maxTimestamp;
|
||||
// Global stats (aggregate only — not used for filtering)
|
||||
cp.last_captured_timestamp = Math.max(cp.last_captured_timestamp, maxTimestamp);
|
||||
cp.total_processed += messageCount;
|
||||
cp.memories_since_last_persona += messageCount;
|
||||
});
|
||||
this.logger.info(
|
||||
`[checkpoint] advanceSessionCapturedTimestamp session=${sessionKey}: -> ${maxTimestamp} ` +
|
||||
`(+${messageCount} msgs), total_processed=${cp.total_processed}`,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Increment L0 conversation count.
|
||||
*/
|
||||
async incrementL0ConversationCount(): Promise<void> {
|
||||
await this.mutate((cp) => {
|
||||
cp.l0_conversations_count += 1;
|
||||
});
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Persona methods (L3)
|
||||
// ============================
|
||||
|
||||
async markPersonaGenerated(totalProcessed: number): Promise<void> {
|
||||
await this.mutate((cp) => {
|
||||
cp.last_persona_at = totalProcessed;
|
||||
cp.last_persona_time = new Date().toISOString();
|
||||
cp.memories_since_last_persona = 0;
|
||||
cp.request_persona_update = false;
|
||||
cp.persona_update_reason = "";
|
||||
});
|
||||
}
|
||||
|
||||
async clearPersonaRequest(): Promise<void> {
|
||||
await this.mutate((cp) => {
|
||||
cp.request_persona_update = false;
|
||||
cp.persona_update_reason = "";
|
||||
});
|
||||
}
|
||||
|
||||
async setPersonaUpdateRequest(reason: string): Promise<void> {
|
||||
await this.mutate((cp) => {
|
||||
cp.request_persona_update = true;
|
||||
cp.persona_update_reason = reason;
|
||||
});
|
||||
}
|
||||
|
||||
async incrementScenesProcessed(): Promise<void> {
|
||||
const cp = await this.mutate((cp) => {
|
||||
cp.scenes_processed += 1;
|
||||
});
|
||||
this.logger.info(`[checkpoint] incrementScenesProcessed: scenes_processed=${cp.scenes_processed}`);
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Per-session helpers — runner state (L0/L1 owned)
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Get or create runner session state for a session.
|
||||
*/
|
||||
getRunnerState(cp: Checkpoint, sessionKey: string): RunnerSessionState {
|
||||
if (!cp.runner_states) {
|
||||
cp.runner_states = {};
|
||||
}
|
||||
let state = cp.runner_states[sessionKey];
|
||||
if (!state) {
|
||||
state = { ...DEFAULT_RUNNER_STATE };
|
||||
cp.runner_states[sessionKey] = state;
|
||||
}
|
||||
return state;
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Per-session helpers — pipeline state (PipelineManager owned)
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Get or create pipeline session state for a session.
|
||||
*/
|
||||
getPipelineState(cp: Checkpoint, sessionKey: string): PipelineSessionState {
|
||||
if (!cp.pipeline_states) {
|
||||
cp.pipeline_states = {};
|
||||
}
|
||||
let state = cp.pipeline_states[sessionKey];
|
||||
if (!state) {
|
||||
state = { ...DEFAULT_PIPELINE_STATE, last_active_time: Date.now() };
|
||||
cp.pipeline_states[sessionKey] = state;
|
||||
}
|
||||
return state;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all pipeline states from checkpoint.
|
||||
*/
|
||||
getAllPipelineStates(cp: Checkpoint): Record<string, PipelineSessionState> {
|
||||
return cp.pipeline_states ?? {};
|
||||
}
|
||||
|
||||
/**
|
||||
* Merge pipeline session states into the checkpoint (used by pipeline persister).
|
||||
* Acquires the file lock so this is safe against concurrent mutations.
|
||||
*
|
||||
* This writes ONLY to `pipeline_states`, never touching `runner_states`.
|
||||
* This is the core guarantee that eliminates the split-brain overwrite bug.
|
||||
*/
|
||||
async mergePipelineStates(states: Record<string, PipelineSessionState>): Promise<void> {
|
||||
await this.mutate((cp) => {
|
||||
if (!cp.pipeline_states) cp.pipeline_states = {};
|
||||
for (const [key, pState] of Object.entries(states)) {
|
||||
cp.pipeline_states[key] = {
|
||||
...cp.pipeline_states[key],
|
||||
...pState,
|
||||
};
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// ============================
|
||||
// L1-specific methods
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Mark L1 extraction completed: reset sinceL1 counter, advance L1 cursor,
|
||||
* and optionally save the last scene name for cross-batch continuity.
|
||||
*/
|
||||
async markL1ExtractionComplete(
|
||||
sessionKey: string,
|
||||
memoriesExtracted: number,
|
||||
cursorTimestamp?: number,
|
||||
lastSceneName?: string,
|
||||
): Promise<void> {
|
||||
await this.mutate((cp) => {
|
||||
const state = this.getRunnerState(cp, sessionKey);
|
||||
if (cursorTimestamp) {
|
||||
state.last_l1_cursor = cursorTimestamp;
|
||||
}
|
||||
if (lastSceneName !== undefined) {
|
||||
state.last_scene_name = lastSceneName;
|
||||
}
|
||||
cp.total_memories_extracted += memoriesExtracted;
|
||||
cp.memories_since_last_persona += memoriesExtracted;
|
||||
});
|
||||
this.logger.info(
|
||||
`[checkpoint] markL1ExtractionComplete session=${sessionKey}: ` +
|
||||
`extracted=${memoriesExtracted}, cursor=${cursorTimestamp ?? "(unchanged)"}, ` +
|
||||
`lastScene="${lastSceneName ?? "(unchanged)"}"`,
|
||||
);
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Atomic capture (race-condition fix)
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Atomically read the per-session cursor, execute the capture callback,
|
||||
* and advance the cursor — all within a single file-lock critical section.
|
||||
*
|
||||
* This eliminates the race window that existed when `read()` (unlocked) and
|
||||
* `advanceSessionCapturedTimestamp()` (locked) were separate calls:
|
||||
* two concurrent `agent_end` events could both read the same stale cursor
|
||||
* and record duplicate messages.
|
||||
*
|
||||
* The callback receives `afterTimestamp` (the current per-session cursor)
|
||||
* and must return either:
|
||||
* - `{ maxTimestamp, messageCount }` to advance the cursor, or
|
||||
* - `null` to leave the cursor unchanged (nothing captured).
|
||||
*
|
||||
* L0 conversation count is also incremented inside the lock when messages
|
||||
* are captured, removing the need for a separate `incrementL0ConversationCount()` call.
|
||||
*
|
||||
* @param sessionKey Per-session identifier
|
||||
* @param pluginStartTimestamp Cold-start floor (used when no cursor exists yet)
|
||||
* @param fn Async callback that performs the actual capture (recordConversation, etc.)
|
||||
*/
|
||||
async captureAtomically(
|
||||
sessionKey: string,
|
||||
pluginStartTimestamp: number | undefined,
|
||||
fn: (afterTimestamp: number) => Promise<{ maxTimestamp: number; messageCount: number } | null>,
|
||||
): Promise<void> {
|
||||
await this.mutate(async (cp) => {
|
||||
// Read the per-session cursor inside the lock
|
||||
const state = this.getRunnerState(cp, sessionKey);
|
||||
let afterTimestamp = state.last_captured_timestamp || 0;
|
||||
|
||||
// Cold-start guard (same logic that was previously in auto-capture.ts)
|
||||
if (afterTimestamp === 0 && pluginStartTimestamp && pluginStartTimestamp > 0) {
|
||||
afterTimestamp = pluginStartTimestamp;
|
||||
}
|
||||
|
||||
const result = await fn(afterTimestamp);
|
||||
|
||||
if (result) {
|
||||
// Advance per-session cursor (runner-owned)
|
||||
state.last_captured_timestamp = result.maxTimestamp;
|
||||
// Global stats (aggregate only — not used for filtering)
|
||||
cp.last_captured_timestamp = Math.max(cp.last_captured_timestamp, result.maxTimestamp);
|
||||
cp.total_processed += result.messageCount;
|
||||
cp.memories_since_last_persona += result.messageCount;
|
||||
// Increment L0 conversation count (was a separate mutate() call before)
|
||||
cp.l0_conversations_count += 1;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,451 @@
|
||||
/**
|
||||
* CleanContextRunner: executes LLM calls in a fully isolated context
|
||||
* using runEmbeddedPiAgent (same mechanism as the llm-task extension).
|
||||
*
|
||||
* Guarantees:
|
||||
* 1. Blank conversation history (temporary session file)
|
||||
* 2. Independent system prompt (only the task prompt)
|
||||
* 3. No tool calls (tools restricted to minimal read-only set to avoid empty tools[] rejection by some providers)
|
||||
* 4. No contamination from the main agent's context
|
||||
*/
|
||||
|
||||
import fs from "node:fs/promises";
|
||||
import fsSync from "node:fs";
|
||||
import path from "node:path";
|
||||
import os from "node:os";
|
||||
import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
import { report } from "../report/reporter.js";
|
||||
|
||||
/**
|
||||
* Resolve a preferred temporary directory for memory-tdai operations.
|
||||
*
|
||||
* Previously imported from `openclaw/plugin-sdk` as `resolvePreferredOpenClawTmpDir`,
|
||||
* but that export was removed in openclaw 2026.2.23+. This local implementation
|
||||
* provides equivalent behavior:
|
||||
* 1. Try `/tmp/openclaw` (if writable)
|
||||
* 2. Fall back to `os.tmpdir()/openclaw-<uid>`
|
||||
*/
|
||||
function resolveOpenClawTmpDir(): string {
|
||||
const POSIX_DIR = "/tmp/openclaw";
|
||||
try {
|
||||
if (fsSync.existsSync(POSIX_DIR)) {
|
||||
fsSync.accessSync(POSIX_DIR, fsSync.constants.W_OK | fsSync.constants.X_OK);
|
||||
return POSIX_DIR;
|
||||
}
|
||||
// Try to create it
|
||||
fsSync.mkdirSync(POSIX_DIR, { recursive: true, mode: 0o700 });
|
||||
return POSIX_DIR;
|
||||
} catch {
|
||||
// Fall back to os.tmpdir()
|
||||
const uid = typeof process.getuid === "function" ? process.getuid() : undefined;
|
||||
const suffix = uid === undefined ? "openclaw" : `openclaw-${uid}`;
|
||||
const fallback = path.join(os.tmpdir(), suffix);
|
||||
fsSync.mkdirSync(fallback, { recursive: true });
|
||||
return fallback;
|
||||
}
|
||||
}
|
||||
|
||||
const TAG = "[memory-tdai] [runner]";
|
||||
|
||||
interface RunnerLogger {
|
||||
debug?: (message: string) => void;
|
||||
info: (message: string) => void;
|
||||
warn: (message: string) => void;
|
||||
error: (message: string) => void;
|
||||
}
|
||||
|
||||
// Dynamic import type — runEmbeddedPiAgent is an internal API
|
||||
type RunEmbeddedPiAgentFn = (params: Record<string, unknown>) => Promise<unknown>;
|
||||
|
||||
// ── Core import (mirrors voice-call/core-bridge.ts — dist/ only, no jiti) ──
|
||||
|
||||
let _rootCache: string | null = null;
|
||||
|
||||
function findPackageRoot(startDir: string, name: string): string | null {
|
||||
let dir = startDir;
|
||||
for (;;) {
|
||||
const pkgPath = path.join(dir, "package.json");
|
||||
try {
|
||||
if (fsSync.existsSync(pkgPath)) {
|
||||
const raw = fsSync.readFileSync(pkgPath, "utf8");
|
||||
const pkg = JSON.parse(raw) as { name?: string };
|
||||
if (pkg.name === name) return dir;
|
||||
}
|
||||
} catch { /* ignore */ }
|
||||
const parent = path.dirname(dir);
|
||||
if (parent === dir) return null;
|
||||
dir = parent;
|
||||
}
|
||||
}
|
||||
|
||||
function resolveOpenClawRoot(): string {
|
||||
if (_rootCache) return _rootCache;
|
||||
const override = process.env.OPENCLAW_ROOT?.trim();
|
||||
if (override) { _rootCache = override; return override; }
|
||||
|
||||
const candidates = new Set<string>();
|
||||
if (process.argv[1]) candidates.add(path.dirname(process.argv[1]));
|
||||
candidates.add(process.cwd());
|
||||
try { candidates.add(path.dirname(fileURLToPath(import.meta.url))); } catch { /* ignore */ }
|
||||
|
||||
for (const start of candidates) {
|
||||
const found = findPackageRoot(start, "openclaw");
|
||||
if (found) { _rootCache = found; return found; }
|
||||
}
|
||||
throw new Error("Unable to resolve OpenClaw root. Set OPENCLAW_ROOT or run `pnpm build`.");
|
||||
}
|
||||
|
||||
let _loadPromise: Promise<RunEmbeddedPiAgentFn> | null = null;
|
||||
|
||||
function loadRunEmbeddedPiAgent(logger?: RunnerLogger): Promise<RunEmbeddedPiAgentFn> {
|
||||
if (_loadPromise) return _loadPromise;
|
||||
|
||||
_loadPromise = (async () => {
|
||||
const t0 = Date.now();
|
||||
const distPath = path.join(resolveOpenClawRoot(), "dist", "extensionAPI.js");
|
||||
if (!fsSync.existsSync(distPath)) {
|
||||
throw new Error(`Missing core module at ${distPath}. Run \`pnpm build\` or install the official package.`);
|
||||
}
|
||||
const mod = await import(pathToFileURL(distPath).href);
|
||||
if (typeof mod.runEmbeddedPiAgent !== "function") {
|
||||
throw new Error("runEmbeddedPiAgent not exported from dist/extensionAPI.js");
|
||||
}
|
||||
logger?.info(`${TAG} loadRunEmbeddedPiAgent: dist/ import OK (${Date.now() - t0}ms)`);
|
||||
return mod.runEmbeddedPiAgent as RunEmbeddedPiAgentFn;
|
||||
})();
|
||||
|
||||
_loadPromise.catch(() => { _loadPromise = null; });
|
||||
return _loadPromise;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pre-warm the embedded agent import. Call this during plugin init to avoid
|
||||
* the cold-start penalty on the first actual extraction run.
|
||||
* Returns immediately (fire-and-forget) — errors are swallowed.
|
||||
*/
|
||||
export function prewarmEmbeddedAgent(logger?: RunnerLogger): void {
|
||||
loadRunEmbeddedPiAgent(logger).catch((err) => {
|
||||
logger?.warn(`${TAG} prewarmEmbeddedAgent: failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
|
||||
});
|
||||
}
|
||||
|
||||
function collectText(payloads: Array<{ text?: string; isError?: boolean }> | undefined): string {
|
||||
const texts = (payloads ?? [])
|
||||
.filter((p) => !p.isError && typeof p.text === "string")
|
||||
.map((p) => p.text ?? "");
|
||||
return texts.join("\n").trim();
|
||||
}
|
||||
|
||||
// ── Model resolution utilities ──
|
||||
|
||||
/** Parsed model reference: { provider, model } */
|
||||
export interface ModelRef {
|
||||
provider: string;
|
||||
model: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a "provider/model" string into its components.
|
||||
* Returns undefined if the input is empty or doesn't contain a "/".
|
||||
*
|
||||
* Examples:
|
||||
* "azure/gpt-5.2-chat" → { provider: "azure", model: "gpt-5.2-chat" }
|
||||
* "custom-host/org/model-v2" → { provider: "custom-host", model: "org/model-v2" }
|
||||
* "" → undefined
|
||||
* "bare-model-name" → undefined (no "/" — may be an alias)
|
||||
*/
|
||||
export function parseModelRef(raw: string | undefined): ModelRef | undefined {
|
||||
if (!raw) return undefined;
|
||||
const trimmed = raw.trim();
|
||||
if (!trimmed) return undefined;
|
||||
|
||||
const slashIdx = trimmed.indexOf("/");
|
||||
if (slashIdx <= 0 || slashIdx === trimmed.length - 1) return undefined;
|
||||
|
||||
return {
|
||||
provider: trimmed.slice(0, slashIdx),
|
||||
model: trimmed.slice(slashIdx + 1),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the user's default model from the main OpenClaw config.
|
||||
*
|
||||
* Resolution order:
|
||||
* 1. Read `agents.defaults.model` (string or { primary })
|
||||
* 2. If the value contains "/", parse directly
|
||||
* 3. If not (may be an alias), look up in `agents.defaults.models` alias table
|
||||
* 4. Return undefined if nothing resolves — let the core use its built-in default
|
||||
*/
|
||||
export function resolveModelFromMainConfig(config: unknown): ModelRef | undefined {
|
||||
if (!config || typeof config !== "object") return undefined;
|
||||
|
||||
const cfg = config as Record<string, unknown>;
|
||||
const agents = cfg.agents as Record<string, unknown> | undefined;
|
||||
if (!agents || typeof agents !== "object") return undefined;
|
||||
|
||||
const defaults = agents.defaults as Record<string, unknown> | undefined;
|
||||
if (!defaults || typeof defaults !== "object") return undefined;
|
||||
|
||||
// Step 1: extract raw model value (string | { primary?: string })
|
||||
const modelCfg = defaults.model;
|
||||
let raw: string | undefined;
|
||||
if (typeof modelCfg === "string") {
|
||||
raw = modelCfg.trim();
|
||||
} else if (modelCfg && typeof modelCfg === "object") {
|
||||
const primary = (modelCfg as Record<string, unknown>).primary;
|
||||
raw = typeof primary === "string" ? primary.trim() : undefined;
|
||||
}
|
||||
if (!raw) return undefined;
|
||||
|
||||
// Step 2: try direct "provider/model" parse
|
||||
const direct = parseModelRef(raw);
|
||||
if (direct) return direct;
|
||||
|
||||
// Step 3: alias lookup — raw doesn't contain "/", check agents.defaults.models
|
||||
const models = defaults.models as Record<string, unknown> | undefined;
|
||||
if (!models || typeof models !== "object") return undefined;
|
||||
|
||||
const rawLower = raw.toLowerCase();
|
||||
for (const [key, entry] of Object.entries(models)) {
|
||||
if (!entry || typeof entry !== "object") continue;
|
||||
const alias = (entry as Record<string, unknown>).alias;
|
||||
if (typeof alias !== "string") continue;
|
||||
if (alias.trim().toLowerCase() !== rawLower) continue;
|
||||
|
||||
// key is "provider/model" format
|
||||
const resolved = parseModelRef(key);
|
||||
if (resolved) return resolved;
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
|
||||
export interface CleanContextRunnerOptions {
|
||||
config: unknown; // OpenClawConfig
|
||||
provider?: string;
|
||||
model?: string;
|
||||
/**
|
||||
* Convenience field: full "provider/model" string.
|
||||
* Takes precedence over separate `provider`/`model` fields.
|
||||
* When all three (modelRef, provider, model) are omitted,
|
||||
* automatically falls back to the main config's `agents.defaults.model`.
|
||||
*/
|
||||
modelRef?: string;
|
||||
/** Allow the LLM to use tools (read_file, write_to_file, etc). Default: false */
|
||||
enableTools?: boolean;
|
||||
/** Logger instance for detailed tracing */
|
||||
logger?: RunnerLogger;
|
||||
}
|
||||
|
||||
// Stable empty directory used as default workspaceDir so that:
|
||||
// 1. Bootstrap/skills scans find nothing → clean LLM context
|
||||
// 2. The path is constant → plugin cacheKey stays stable (no re-registration)
|
||||
let _cleanWorkspaceDir: string | undefined;
|
||||
async function getCleanWorkspaceDir(): Promise<string> {
|
||||
if (_cleanWorkspaceDir) return _cleanWorkspaceDir;
|
||||
const dir = path.join(resolveOpenClawTmpDir(), "memory-tdai-clean-workspace");
|
||||
await fs.mkdir(dir, { recursive: true });
|
||||
_cleanWorkspaceDir = dir;
|
||||
return dir;
|
||||
}
|
||||
|
||||
export class CleanContextRunner {
|
||||
private options: CleanContextRunnerOptions;
|
||||
private logger: RunnerLogger | undefined;
|
||||
/** Resolved provider after modelRef / config fallback */
|
||||
private resolvedProvider: string | undefined;
|
||||
/** Resolved model after modelRef / config fallback */
|
||||
private resolvedModel: string | undefined;
|
||||
|
||||
constructor(options: CleanContextRunnerOptions) {
|
||||
this.options = options;
|
||||
this.logger = options.logger;
|
||||
|
||||
// Model resolution priority:
|
||||
// 1. modelRef ("provider/model" string) — highest
|
||||
// 2. explicit provider + model fields
|
||||
// 3. main config agents.defaults.model — automatic fallback
|
||||
// 4. undefined (let core use built-in default)
|
||||
const fromRef = parseModelRef(options.modelRef);
|
||||
if (fromRef) {
|
||||
this.resolvedProvider = fromRef.provider;
|
||||
this.resolvedModel = fromRef.model;
|
||||
} else if (options.provider || options.model) {
|
||||
this.resolvedProvider = options.provider;
|
||||
this.resolvedModel = options.model;
|
||||
} else {
|
||||
// No explicit model specified — fall back to main config
|
||||
const fromConfig = resolveModelFromMainConfig(options.config);
|
||||
if (fromConfig) {
|
||||
this.resolvedProvider = fromConfig.provider;
|
||||
this.resolvedModel = fromConfig.model;
|
||||
this.logger?.debug?.(
|
||||
`${TAG} Using model from main config: ${fromConfig.provider}/${fromConfig.model}`,
|
||||
);
|
||||
}
|
||||
// else: both undefined → core will use its built-in default (anthropic/claude-opus-4-6)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Run a prompt in a fully isolated clean context.
|
||||
* Returns the LLM's text output.
|
||||
*
|
||||
* When `workspaceDir` is provided it overrides the default `process.cwd()`,
|
||||
* letting the LLM's file-tool calls resolve paths relative to a custom root.
|
||||
*/
|
||||
async run(params: {
|
||||
prompt: string;
|
||||
/** Optional system prompt. When provided, `prompt` is used as the user message. */
|
||||
systemPrompt?: string;
|
||||
taskId: string;
|
||||
timeoutMs?: number;
|
||||
maxTokens?: number;
|
||||
workspaceDir?: string;
|
||||
/** Plugin instance ID for llm_call metric (optional) */
|
||||
instanceId?: string;
|
||||
}): Promise<string> {
|
||||
const runStartMs = Date.now();
|
||||
this.logger?.debug?.(`${TAG} run() start: taskId=${params.taskId}, timeout=${params.timeoutMs ?? 120_000}ms, tools=${this.options.enableTools ? "enabled" : "disabled"}, workspaceDir=${params.workspaceDir ?? "(default)"}`);
|
||||
|
||||
const tmpDir = await fs.mkdtemp(
|
||||
path.join(resolveOpenClawTmpDir(), `memory-tdai-${params.taskId}-`),
|
||||
);
|
||||
const cleanWorkspace = params.workspaceDir ?? await getCleanWorkspaceDir();
|
||||
this.logger?.debug?.(`${TAG} run() tmpDir=${tmpDir}, cleanWorkspace=${cleanWorkspace}`);
|
||||
|
||||
try {
|
||||
const sessionFile = path.join(tmpDir, "session.json");
|
||||
|
||||
// Phase 1: Load runEmbeddedPiAgent (fast if dist/ exists or already cached)
|
||||
const importStartMs = Date.now();
|
||||
const runEmbeddedPiAgent = await loadRunEmbeddedPiAgent(this.logger);
|
||||
const importElapsedMs = Date.now() - importStartMs;
|
||||
this.logger?.debug?.(`${TAG} run() dynamic import phase: ${importElapsedMs}ms`);
|
||||
|
||||
// Derive a config with plugins disabled to prevent loadOpenClawPlugins
|
||||
// from re-registering plugins when the workspaceDir differs from the
|
||||
// gateway's original workspace (cacheKey mismatch triggers full reload).
|
||||
//
|
||||
// Security: restrict available tools to the minimal set needed for
|
||||
// scene extraction (read/write/edit). This prevents the LLM from
|
||||
// accessing exec, sessions, browser, cron, or any other powerful tools.
|
||||
// File deletion is handled via "soft-delete" (write empty) + cleanup afterward.
|
||||
const cleanConfig = {
|
||||
...(this.options.config as Record<string, unknown>),
|
||||
plugins: {
|
||||
...((this.options.config as Record<string, unknown>)?.plugins as Record<string, unknown> | undefined),
|
||||
enabled: false,
|
||||
},
|
||||
tools: {
|
||||
...((this.options.config as Record<string, unknown>)?.tools as Record<string, unknown> | undefined),
|
||||
// When enableTools=false we still keep one lightweight read-only tool
|
||||
// so that the tools array sent to the API is non-empty.
|
||||
// Some providers (e.g. qwencode) reject tools:[] with minItems:1 validation.
|
||||
allow: this.options.enableTools ? ["read", "write", "edit"] : ["read"],
|
||||
},
|
||||
};
|
||||
|
||||
// Build the effective prompt:
|
||||
// If systemPrompt is provided, pass it as a separate parameter to the agent
|
||||
// and use `prompt` as the user message. Fallback: prepend to prompt if the
|
||||
// embedded agent doesn't support systemPrompt natively.
|
||||
const effectivePrompt = params.systemPrompt
|
||||
? `${params.systemPrompt}\n\n---\n\n${params.prompt}`
|
||||
: params.prompt;
|
||||
|
||||
const ts = Date.now();
|
||||
const sessionId = `memory-${params.taskId}-session-${ts}`;
|
||||
const runId = `memory-${params.taskId}-run-${ts}`;
|
||||
this.logger?.debug?.(`${TAG} run() starting embedded agent: sessionId=${sessionId}, runId=${runId}, provider=${this.resolvedProvider ?? "(default)"}, model=${this.resolvedModel ?? "(default)"}`);
|
||||
|
||||
// Phase 2: Embedded agent run (LLM call + tool calls)
|
||||
const agentStartMs = Date.now();
|
||||
const result = await runEmbeddedPiAgent({
|
||||
sessionId,
|
||||
sessionFile,
|
||||
workspaceDir: cleanWorkspace,
|
||||
config: cleanConfig,
|
||||
prompt: effectivePrompt,
|
||||
systemPrompt: params.systemPrompt,
|
||||
timeoutMs: params.timeoutMs ?? 120_000,
|
||||
runId,
|
||||
provider: this.resolvedProvider,
|
||||
model: this.resolvedModel,
|
||||
// Do NOT pass disableTools:true — that produces tools:[] which some
|
||||
// providers (qwencode) reject with "[] is too short - 'tools'".
|
||||
// Instead rely on cleanConfig.tools.allow to restrict the tool set
|
||||
// to a minimal read-only tool (when enableTools=false).
|
||||
disableTools: false,
|
||||
streamParams: {
|
||||
maxTokens: params.maxTokens,
|
||||
},
|
||||
});
|
||||
const agentElapsedMs = Date.now() - agentStartMs;
|
||||
this.logger?.debug?.(`${TAG} run() embedded agent completed: ${agentElapsedMs}ms`);
|
||||
|
||||
// Phase 3: Collect output
|
||||
const text = collectText((result as Record<string, unknown>).payloads as Array<{ text?: string; isError?: boolean }> | undefined);
|
||||
const totalMs = Date.now() - runStartMs;
|
||||
|
||||
if (!text) {
|
||||
// Empty output is normal when the LLM decides there is nothing to
|
||||
// extract (e.g. trivial greetings). Log a warning instead of
|
||||
// throwing so the caller can handle it gracefully.
|
||||
this.logger?.warn?.(`${TAG} run() empty output after ${totalMs}ms (import=${importElapsedMs}ms, agent=${agentElapsedMs}ms) — treating as empty result`);
|
||||
// llm_call metric (empty output)
|
||||
if (params.instanceId && this.logger) {
|
||||
report("llm_call", {
|
||||
taskId: params.taskId,
|
||||
provider: this.resolvedProvider ?? "default",
|
||||
model: this.resolvedModel ?? "default",
|
||||
inputLength: params.prompt.length,
|
||||
outputLength: 0,
|
||||
totalDurationMs: totalMs,
|
||||
success: true,
|
||||
error: "empty_output",
|
||||
});
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
this.logger?.debug?.(`${TAG} run() completed: ${totalMs}ms total (import=${importElapsedMs}ms, agent=${agentElapsedMs}ms), output=${text.length} chars`);
|
||||
|
||||
// ── llm_call metric (success) ──
|
||||
if (params.instanceId && this.logger) {
|
||||
report("llm_call", {
|
||||
taskId: params.taskId,
|
||||
provider: this.resolvedProvider ?? "default",
|
||||
model: this.resolvedModel ?? "default",
|
||||
inputLength: params.prompt.length,
|
||||
outputLength: text.length,
|
||||
totalDurationMs: totalMs,
|
||||
success: true,
|
||||
error: null,
|
||||
});
|
||||
}
|
||||
|
||||
return text;
|
||||
} catch (err) {
|
||||
const totalMs = Date.now() - runStartMs;
|
||||
this.logger?.error(`${TAG} run() failed after ${totalMs}ms: ${err instanceof Error ? err.stack ?? err.message : String(err)}`);
|
||||
// ── llm_call metric (failure) ──
|
||||
if (params.instanceId && this.logger) {
|
||||
report("llm_call", {
|
||||
taskId: params.taskId,
|
||||
provider: this.resolvedProvider ?? "default",
|
||||
model: this.resolvedModel ?? "default",
|
||||
inputLength: params.prompt.length,
|
||||
outputLength: 0,
|
||||
totalDurationMs: totalMs,
|
||||
success: false,
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
});
|
||||
}
|
||||
throw err;
|
||||
} finally {
|
||||
await fs.rm(tmpDir, { recursive: true, force: true }).catch(() => {});
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
/**
|
||||
* ManagedTimer: a named, lifecycle-managed wrapper around setTimeout.
|
||||
*
|
||||
* Eliminates repetitive clear→set→fire→clean patterns by providing:
|
||||
* - `schedule(delayMs, cb)` — cancel any pending timer, set a new one
|
||||
* - `scheduleAt(epochMs, cb)` — schedule by absolute time point
|
||||
* - `tryAdvanceTo(epochMs, cb)` — only reschedule if new time is *earlier*
|
||||
* - `cancel()` — cancel without triggering
|
||||
* - `flush()` — trigger immediately (for graceful shutdown)
|
||||
* - `pending` — whether a timer is waiting
|
||||
*
|
||||
* The optional `isDestroyed` guard prevents firing after the owner is torn down.
|
||||
*/
|
||||
|
||||
type TimerHandle = ReturnType<typeof setTimeout>;
|
||||
|
||||
export class ManagedTimer {
|
||||
private handle: TimerHandle | null = null;
|
||||
private callback: (() => void) | null = null;
|
||||
/** Absolute epoch-ms when the current timer is scheduled to fire. */
|
||||
private scheduledAt = 0;
|
||||
|
||||
constructor(
|
||||
/** Human-readable name for logging. */
|
||||
public readonly name: string,
|
||||
/** If provided, checked before firing — skips callback when true. */
|
||||
private readonly isDestroyed?: () => boolean,
|
||||
) {}
|
||||
|
||||
// ── Core operations ──────────────────────────────────
|
||||
|
||||
/**
|
||||
* Cancel any pending timer and schedule a new one after `delayMs`.
|
||||
* The callback fires once; the timer auto-clears after firing.
|
||||
*/
|
||||
schedule(delayMs: number, callback: () => void): void {
|
||||
this.cancelInternal();
|
||||
this.callback = callback;
|
||||
this.scheduledAt = Date.now() + delayMs;
|
||||
this.handle = setTimeout(() => this.fire(), delayMs);
|
||||
// Don't let pipeline timers keep the process alive in CLI mode.
|
||||
// In gateway mode the server listener holds the event loop anyway.
|
||||
this.handle.unref();
|
||||
}
|
||||
|
||||
/**
|
||||
* Cancel any pending timer and schedule to fire at an absolute epoch-ms.
|
||||
* If `epochMs` is in the past, fires on next tick (delay = 0).
|
||||
*/
|
||||
scheduleAt(epochMs: number, callback: () => void): void {
|
||||
this.cancelInternal();
|
||||
this.callback = callback;
|
||||
this.scheduledAt = epochMs;
|
||||
const delay = Math.max(0, epochMs - Date.now());
|
||||
this.handle = setTimeout(() => this.fire(), delay);
|
||||
this.handle.unref();
|
||||
}
|
||||
|
||||
/**
|
||||
* Only reschedule if `epochMs` is *earlier* than the current scheduled time.
|
||||
* This implements the "downward-only" timer pattern (L2 scheduling).
|
||||
* If no timer is pending, behaves like `scheduleAt()`.
|
||||
*
|
||||
* @returns true if the timer was actually advanced (or newly set).
|
||||
*/
|
||||
tryAdvanceTo(epochMs: number, callback: () => void): boolean {
|
||||
if (this.handle === null) {
|
||||
// No pending timer → set it
|
||||
this.scheduleAt(epochMs, callback);
|
||||
return true;
|
||||
}
|
||||
|
||||
if (epochMs < this.scheduledAt) {
|
||||
// New time is earlier → reschedule
|
||||
this.scheduleAt(epochMs, callback);
|
||||
return true;
|
||||
}
|
||||
|
||||
// Current timer is already earlier or equal → keep it
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Cancel the pending timer without triggering the callback.
|
||||
*/
|
||||
cancel(): void {
|
||||
this.cancelInternal();
|
||||
}
|
||||
|
||||
/**
|
||||
* Immediately trigger the callback (if pending) and clear the timer.
|
||||
* Used for graceful shutdown to flush pending work.
|
||||
*
|
||||
* Note: Unlike `fire()`, this method intentionally does NOT check `isDestroyed`.
|
||||
* This is by design — during shutdown, `destroy()` sets `destroyed = true` first,
|
||||
* then calls `flush()` to drain pending work. The `isDestroyed` guard only applies
|
||||
* to natural timer expiration via `fire()`, not to explicit shutdown flushes.
|
||||
*/
|
||||
flush(): void {
|
||||
if (this.handle === null) return;
|
||||
const cb = this.callback;
|
||||
this.cancelInternal();
|
||||
if (cb) cb();
|
||||
}
|
||||
|
||||
// ── Accessors ────────────────────────────────────────
|
||||
|
||||
/** Whether a timer is currently pending. */
|
||||
get pending(): boolean {
|
||||
return this.handle !== null;
|
||||
}
|
||||
|
||||
/** The epoch-ms when the current timer is scheduled to fire (0 if none). */
|
||||
get scheduledTime(): number {
|
||||
return this.handle !== null ? this.scheduledAt : 0;
|
||||
}
|
||||
|
||||
// ── Internals ────────────────────────────────────────
|
||||
|
||||
private fire(): void {
|
||||
const cb = this.callback;
|
||||
this.handle = null;
|
||||
this.callback = null;
|
||||
this.scheduledAt = 0;
|
||||
|
||||
if (this.isDestroyed?.()) return;
|
||||
if (cb) cb();
|
||||
}
|
||||
|
||||
private cancelInternal(): void {
|
||||
if (this.handle !== null) {
|
||||
clearTimeout(this.handle);
|
||||
this.handle = null;
|
||||
}
|
||||
this.callback = null;
|
||||
this.scheduledAt = 0;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,331 @@
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
|
||||
import type { VectorStore } from "../store/vector-store.js";
|
||||
import { ManagedTimer } from "./managed-timer.js";
|
||||
|
||||
interface Logger {
|
||||
debug?: (message: string) => void;
|
||||
info: (message: string) => void;
|
||||
warn: (message: string) => void;
|
||||
error: (message: string) => void;
|
||||
}
|
||||
|
||||
export interface MemoryCleanerOptions {
|
||||
baseDir: string;
|
||||
retentionDays: number;
|
||||
cleanTime: string;
|
||||
logger?: Logger;
|
||||
vectorStore?: VectorStore;
|
||||
}
|
||||
|
||||
interface CleanupStats {
|
||||
scannedFiles: number;
|
||||
changedFiles: number;
|
||||
skippedNonShardFiles: number;
|
||||
deleteFailedFiles: number;
|
||||
}
|
||||
|
||||
const TAG = "[memory-tdai][cleaner]";
|
||||
const L0_DIR_NAME = "conversations";
|
||||
const L1_DIR_NAME = "records";
|
||||
|
||||
export class LocalMemoryCleaner {
|
||||
private readonly timer: ManagedTimer;
|
||||
private destroyed = false;
|
||||
private vectorStore?: VectorStore;
|
||||
|
||||
constructor(private readonly opts: MemoryCleanerOptions) {
|
||||
this.timer = new ManagedTimer("memory-tdai-cleaner", () => this.destroyed);
|
||||
this.vectorStore = opts.vectorStore;
|
||||
}
|
||||
|
||||
setVectorStore(vectorStore: VectorStore | undefined): void {
|
||||
this.vectorStore = vectorStore;
|
||||
}
|
||||
|
||||
start(): void {
|
||||
if (this.destroyed) return;
|
||||
|
||||
const now = new Date();
|
||||
const tz = Intl.DateTimeFormat().resolvedOptions().timeZone || "unknown";
|
||||
const utcOffset = formatUtcOffset(-now.getTimezoneOffset());
|
||||
|
||||
this.opts.logger?.info(
|
||||
`${TAG} Enabled: retentionDays=${this.opts.retentionDays}, cleanTime=${this.opts.cleanTime}, dirs=[${L0_DIR_NAME}, ${L1_DIR_NAME}]`,
|
||||
);
|
||||
this.opts.logger?.info(
|
||||
`${TAG} Runtime clock: nowLocal=${formatLocalDateTime(now)}, nowIso=${now.toISOString()}, tz=${tz}, utcOffset=${utcOffset}`,
|
||||
);
|
||||
|
||||
this.scheduleNext();
|
||||
}
|
||||
|
||||
destroy(): void {
|
||||
if (this.destroyed) return;
|
||||
this.destroyed = true;
|
||||
this.timer.cancel();
|
||||
this.opts.logger?.info(`${TAG} Stopped`);
|
||||
}
|
||||
|
||||
async runOnce(nowMs = Date.now()): Promise<void> {
|
||||
if (this.destroyed) return;
|
||||
|
||||
const retentionDays = this.opts.retentionDays;
|
||||
if (!(retentionDays > 0)) {
|
||||
this.opts.logger?.debug?.(`${TAG} Skip run: invalid retentionDays=${retentionDays}`);
|
||||
return;
|
||||
}
|
||||
|
||||
// 按“本地自然日”保留策略计算截止时间。
|
||||
// 例如 retentionDays=2，今天是 03-15，则保留 03-14/03-15，删除早于 03-14 00:00:00.000 的记录。
|
||||
const cutoffMs = computeCutoffMsByLocalDay(nowMs, retentionDays);
|
||||
const targetDirs = [
|
||||
path.join(this.opts.baseDir, L0_DIR_NAME),
|
||||
path.join(this.opts.baseDir, L1_DIR_NAME),
|
||||
];
|
||||
|
||||
const total: CleanupStats = {
|
||||
scannedFiles: 0,
|
||||
changedFiles: 0,
|
||||
skippedNonShardFiles: 0,
|
||||
deleteFailedFiles: 0,
|
||||
};
|
||||
|
||||
for (const dirPath of targetDirs) {
|
||||
const stats = await this.cleanDirectory(dirPath, cutoffMs);
|
||||
total.scannedFiles += stats.scannedFiles;
|
||||
total.changedFiles += stats.changedFiles;
|
||||
total.skippedNonShardFiles += stats.skippedNonShardFiles;
|
||||
total.deleteFailedFiles += stats.deleteFailedFiles;
|
||||
}
|
||||
|
||||
if (this.vectorStore) {
|
||||
const vectorStore = this.vectorStore;
|
||||
const cutoffIso = new Date(cutoffMs).toISOString();
|
||||
|
||||
let removedL0 = 0;
|
||||
let removedL1 = 0;
|
||||
let failedL0DbCleanup = 0;
|
||||
let failedL1DbCleanup = 0;
|
||||
|
||||
try {
|
||||
removedL0 = vectorStore.deleteL0ExpiredByRecordedAt(cutoffIso);
|
||||
} catch (err) {
|
||||
failedL0DbCleanup = 1;
|
||||
this.opts.logger?.warn(
|
||||
`${TAG} SQLite cleanup L0 failed: ${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
removedL1 = vectorStore.deleteL1ExpiredByUpdatedTime(cutoffIso);
|
||||
} catch (err) {
|
||||
failedL1DbCleanup = 1;
|
||||
this.opts.logger?.warn(
|
||||
`${TAG} SQLite cleanup L1 failed: ${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
}
|
||||
|
||||
if (removedL1 > 0 || removedL0 > 0) {
|
||||
total.changedFiles += 1;
|
||||
}
|
||||
|
||||
this.opts.logger?.info(
|
||||
`${TAG} SQLite cleanup done: removedL1Records=${removedL1}, removedL0Records=${removedL0}, failedL1DbCleanup=${failedL1DbCleanup}, failedL0DbCleanup=${failedL0DbCleanup}, cutoffIso=${cutoffIso}`,
|
||||
);
|
||||
}
|
||||
|
||||
this.opts.logger?.info(
|
||||
`${TAG} Cleanup done: scannedFiles=${total.scannedFiles}, changedFiles=${total.changedFiles}, skippedNonShardFiles=${total.skippedNonShardFiles}, deleteFailedFiles=${total.deleteFailedFiles}`,
|
||||
);
|
||||
|
||||
}
|
||||
|
||||
private scheduleNext(): void {
|
||||
const nowMs = Date.now();
|
||||
const now = new Date(nowMs);
|
||||
const next = nextRunAt(this.opts.cleanTime, nowMs);
|
||||
const targetToday = buildTodayRunTime(this.opts.cleanTime, nowMs);
|
||||
const passedToday = targetToday <= nowMs;
|
||||
const delayMs = Math.max(0, next - nowMs);
|
||||
|
||||
this.opts.logger?.info(
|
||||
`${TAG} Schedule next run: nowLocal=${formatLocalDateTime(now)}, cleanTime=${this.opts.cleanTime}, targetTodayLocal=${formatLocalDateTime(new Date(targetToday))}, passedToday=${passedToday}, nextRunLocal=${formatLocalDateTime(new Date(next))}, nextRunIso=${new Date(next).toISOString()}, delayMs=${delayMs}`,
|
||||
);
|
||||
|
||||
this.timer.scheduleAt(next, () => {
|
||||
const firedAtMs = Date.now();
|
||||
this.opts.logger?.info(
|
||||
`${TAG} Timer fired: scheduledLocal=${formatLocalDateTime(new Date(next))}, firedLocal=${formatLocalDateTime(new Date(firedAtMs))}, driftMs=${firedAtMs - next}`,
|
||||
);
|
||||
void this.runAndReschedule();
|
||||
});
|
||||
}
|
||||
|
||||
private async runAndReschedule(): Promise<void> {
|
||||
if (this.destroyed) return;
|
||||
const runStart = new Date();
|
||||
this.opts.logger?.info(
|
||||
`${TAG} Cleanup tick start: nowLocal=${formatLocalDateTime(runStart)}, nowIso=${runStart.toISOString()}`,
|
||||
);
|
||||
|
||||
try {
|
||||
await this.runOnce();
|
||||
} catch (err) {
|
||||
this.opts.logger?.error(`${TAG} Cleanup failed: ${err instanceof Error ? err.stack ?? err.message : String(err)}`);
|
||||
} finally {
|
||||
if (!this.destroyed) {
|
||||
this.scheduleNext();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async cleanDirectory(dirPath: string, cutoffMs: number): Promise<CleanupStats> {
|
||||
const stats: CleanupStats = {
|
||||
scannedFiles: 0,
|
||||
changedFiles: 0,
|
||||
skippedNonShardFiles: 0,
|
||||
deleteFailedFiles: 0,
|
||||
};
|
||||
|
||||
let entries;
|
||||
try {
|
||||
entries = await fs.readdir(dirPath, { withFileTypes: true });
|
||||
} catch {
|
||||
|
||||
this.opts.logger?.debug?.(`${TAG} Directory not found, skip: ${dirPath}`);
|
||||
return stats;
|
||||
}
|
||||
|
||||
for (const entry of entries) {
|
||||
if (!entry.isFile()) continue;
|
||||
if (!isJsonLikeFile(entry.name)) continue;
|
||||
|
||||
const filePath = path.join(dirPath, entry.name);
|
||||
stats.scannedFiles += 1;
|
||||
|
||||
// 仅支持日期分片文件：YYYY-MM-DD(.jsonl/.json)
|
||||
const shard = extractShardDateFromFileName(entry.name);
|
||||
if (!shard) {
|
||||
stats.skippedNonShardFiles += 1;
|
||||
this.opts.logger?.debug?.(`${TAG} Skip non-shard file: ${filePath}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const dayEndMs = localDayEndMs(shard.year, shard.month, shard.day);
|
||||
if (dayEndMs < cutoffMs) {
|
||||
try {
|
||||
await fs.unlink(filePath);
|
||||
stats.changedFiles += 1;
|
||||
this.opts.logger?.info(`${TAG} Removed expired file by name: ${filePath}`);
|
||||
} catch (err) {
|
||||
stats.deleteFailedFiles += 1;
|
||||
this.opts.logger?.warn(
|
||||
`${TAG} Failed to delete expired shard file ${filePath}: ${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
}
|
||||
} else {
|
||||
this.opts.logger?.debug?.(`${TAG} Keep shard file by name: ${filePath}`);
|
||||
}
|
||||
}
|
||||
|
||||
return stats;
|
||||
}
|
||||
}
|
||||
|
||||
function isJsonLikeFile(name: string): boolean {
|
||||
return name.endsWith(".jsonl") || name.endsWith(".json");
|
||||
}
|
||||
|
||||
function extractShardDateFromFileName(
|
||||
fileName: string,
|
||||
): { year: number; month: number; day: number } | undefined {
|
||||
|
||||
// Supported format: YYYY-MM-DD.jsonl | YYYY-MM-DD.json
|
||||
const m = /^(\d{4})-(\d{2})-(\d{2})\.(?:jsonl|json)$/.exec(fileName);
|
||||
if (!m) return undefined;
|
||||
|
||||
const year = Number(m[1]);
|
||||
const month = Number(m[2]);
|
||||
const day = Number(m[3]);
|
||||
|
||||
if (!Number.isInteger(year) || !Number.isInteger(month) || !Number.isInteger(day)) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
if (month < 1 || month > 12 || day < 1 || day > 31) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const probe = new Date(year, month - 1, day);
|
||||
if (
|
||||
probe.getFullYear() !== year
|
||||
|| probe.getMonth() !== month - 1
|
||||
|| probe.getDate() !== day
|
||||
) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return { year, month, day };
|
||||
}
|
||||
|
||||
function localDayEndMs(year: number, month: number, day: number): number {
|
||||
const end = new Date(year, month - 1, day, 23, 59, 59, 999);
|
||||
return end.getTime();
|
||||
}
|
||||
|
||||
function formatLocalDateTime(d: Date): string {
|
||||
const y = d.getFullYear();
|
||||
const m = String(d.getMonth() + 1).padStart(2, "0");
|
||||
const day = String(d.getDate()).padStart(2, "0");
|
||||
const hh = String(d.getHours()).padStart(2, "0");
|
||||
const mm = String(d.getMinutes()).padStart(2, "0");
|
||||
const ss = String(d.getSeconds()).padStart(2, "0");
|
||||
return `${y}-${m}-${day} ${hh}:${mm}:${ss}`;
|
||||
}
|
||||
|
||||
function formatUtcOffset(offsetMinutes: number): string {
|
||||
const sign = offsetMinutes >= 0 ? "+" : "-";
|
||||
const abs = Math.abs(offsetMinutes);
|
||||
const hh = String(Math.floor(abs / 60)).padStart(2, "0");
|
||||
const mm = String(abs % 60).padStart(2, "0");
|
||||
return `${sign}${hh}:${mm}`;
|
||||
}
|
||||
|
||||
function computeCutoffMsByLocalDay(nowMs: number, retentionDays: number): number {
|
||||
// 自然日策略，保留“今天 + 往前 retentionDays-1 天”
|
||||
// 删除阈值为 keepStart 当天 00:00:00.000（本地时区）
|
||||
const now = new Date(nowMs);
|
||||
const keepStart = new Date(now.getFullYear(), now.getMonth(), now.getDate(), 0, 0, 0, 0);
|
||||
keepStart.setDate(keepStart.getDate() - (retentionDays - 1));
|
||||
return keepStart.getTime();
|
||||
}
|
||||
|
||||
function buildTodayRunTime(cleanTime: string, nowMs: number): number {
|
||||
|
||||
const [hRaw, mRaw] = cleanTime.split(":");
|
||||
const hour = Number(hRaw);
|
||||
const minute = Number(mRaw);
|
||||
|
||||
const target = new Date(nowMs);
|
||||
target.setHours(hour, minute, 0, 0);
|
||||
return target.getTime();
|
||||
}
|
||||
|
||||
function nextRunAt(cleanTime: string, nowMs: number): number {
|
||||
|
||||
const [hRaw, mRaw] = cleanTime.split(":");
|
||||
const hour = Number(hRaw);
|
||||
const minute = Number(mRaw);
|
||||
|
||||
const now = new Date(nowMs);
|
||||
const next = new Date(nowMs);
|
||||
next.setHours(hour, minute, 0, 0);
|
||||
|
||||
if (next.getTime() <= now.getTime()) {
|
||||
next.setDate(next.getDate() + 1);
|
||||
}
|
||||
|
||||
return next.getTime();
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,398 @@
|
||||
/**
|
||||
* Text sanitization for memory pipeline (capture & recall).
|
||||
* Removes injected tags, gateway metadata, media noise, etc.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Clean text for the memory pipeline: remove injected tags, metadata,
|
||||
* timestamps, media markers and base64 image data.
|
||||
*
|
||||
* Used by both capture (L0 recording) and recall (query cleaning) paths.
|
||||
*/
|
||||
export function sanitizeText(text: string): string {
|
||||
let cleaned = text;
|
||||
|
||||
// Remove injected memory context tags (prevent feedback loops)
|
||||
cleaned = cleaned.replace(/<relevant-memories>[\s\S]*?<\/relevant-memories>/g, "");
|
||||
cleaned = cleaned.replace(/<user-persona>[\s\S]*?<\/user-persona>/g, "");
|
||||
cleaned = cleaned.replace(/<relevant-scenes>[\s\S]*?<\/relevant-scenes>/g, "");
|
||||
cleaned = cleaned.replace(/<scene-navigation>[\s\S]*?<\/scene-navigation>/g, "");
|
||||
|
||||
// Remove framework-injected inbound metadata blocks (from inbound-meta.ts buildInboundUserContextPrefix).
|
||||
// These are "label:\n```json\n...\n```" blocks that the framework prepends to user messages.
|
||||
// Pattern matches all known block labels:
|
||||
// - Conversation info (untrusted metadata):
|
||||
// - Sender (untrusted metadata):
|
||||
// - Thread starter (untrusted, for context):
|
||||
// - Replied message (untrusted, for context):
|
||||
// - Forwarded message context (untrusted metadata):
|
||||
// - Chat history since last reply (untrusted, for context):
|
||||
cleaned = cleaned.replace(
|
||||
/(?:Conversation info|Sender|Thread starter|Replied message|Forwarded message context|Chat history since last reply)\s*\(untrusted[\s\S]*?\):\s*```json\s*[\s\S]*?```/g,
|
||||
"",
|
||||
);
|
||||
|
||||
// Remove conversation metadata JSON blocks (legacy pattern)
|
||||
cleaned = cleaned.replace(/```json\s*\{[\s\S]*?"session[\s\S]*?\}\s*```/g, "");
|
||||
|
||||
// Remove framework reply directive tags: [[reply_to_current]], [[reply_to_xxx]], etc.
|
||||
cleaned = cleaned.replace(/\[\[reply_to[^\]]*\]\]\s*/g, "");
|
||||
|
||||
// Remove line-leading timestamps, e.g. "[Tue 2026-03-24 03:48 UTC]"
|
||||
// or "[Tue 2026-03-24 20:21 GMT+8]", "[Thu 2026-03-24 01:51 GMT+5:30]"
|
||||
// Matches brackets containing word chars, digits, hyphens, colons, plus signs,
|
||||
// and spaces — the '+' is needed for timezone offsets like GMT+8, GMT+5:30.
|
||||
cleaned = cleaned.replace(/^\[[\w\d\-:+ ]+\]\s*/gm, "");
|
||||
|
||||
// Remove gateway media-attachment markers:
|
||||
// [media attached: /path/to/file.png (image/png) | /path/to/file.png]
|
||||
cleaned = cleaned.replace(/\[media attached:[^\]]*\]\s*/g, "");
|
||||
|
||||
// Remove gateway image-reply instructions injected after media attachments.
|
||||
// Starts with "To send an image back" and ends before the next real content.
|
||||
cleaned = cleaned.replace(
|
||||
/To send an image back,[\s\S]*?(?:Keep caption in the text body\.)\s*/g,
|
||||
"",
|
||||
);
|
||||
|
||||
// Remove "System: [timestamp] Exec completed ..." blocks appended by the framework.
|
||||
cleaned = cleaned.replace(/^System:\s*\[[\s\S]*?$/gm, "");
|
||||
|
||||
// Remove inline base64 image data URIs (e.g. data:image/png;base64,iVBOR...)
|
||||
// Replace with empty string (not a placeholder) so that pure-image messages
|
||||
// become empty after sanitization and are naturally filtered by length checks.
|
||||
cleaned = cleaned.replace(/data:image\/[a-z+]+;base64,[A-Za-z0-9+/=]+/gi, "");
|
||||
|
||||
// Remove null chars + compress whitespace
|
||||
cleaned = cleaned.replace(/\0/g, "").replace(/\n{3,}/g, "\n\n").trim();
|
||||
|
||||
return cleaned;
|
||||
}
|
||||
|
||||
/**
|
||||
* Strip fenced code blocks from assistant replies before L0 capture.
|
||||
*
|
||||
* AI responses often contain large code snippets (```...```) that dilute
|
||||
* the semantic signal for embedding and memory extraction. This function
|
||||
* removes only the code block content while preserving surrounding
|
||||
* natural-language explanations.
|
||||
*
|
||||
* Only applied to `role=assistant` messages in the L0 capture path —
|
||||
* user messages and recall queries are NOT affected.
|
||||
*/
|
||||
export function stripCodeBlocks(text: string): string {
|
||||
return text.replace(/```[^\n]*\n[\s\S]*?```/g, "").replace(/\n{3,}/g, "\n\n").trim();
|
||||
}
|
||||
|
||||
// ============================
|
||||
// L0 / L1 Capture & Extraction Filters
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* L0 capture filter — intentionally **permissive**.
|
||||
*
|
||||
* L0 is the raw conversation archive. We want to preserve as much user input
|
||||
* as possible so that downstream stages (L1 extraction, search, analytics)
|
||||
* have the full picture. Only messages that are *structurally* useless are
|
||||
* dropped here:
|
||||
* - Empty / whitespace-only text
|
||||
* - Framework-internal noise (bootstrap, session reset, NO_REPLY, …)
|
||||
* - Slash commands (/new, /reset, …)
|
||||
*
|
||||
* Content-quality filters (length, symbols, prompt injection) are deferred
|
||||
* to {@link shouldExtractL1}.
|
||||
*/
|
||||
export function shouldCaptureL0(text: string): boolean {
|
||||
if (!text || !text.trim()) return false;
|
||||
|
||||
// Filter framework-internal / bootstrap noise messages
|
||||
if (isFrameworkNoise(text)) return false;
|
||||
|
||||
// Slash commands are framework directives, not user content
|
||||
if (text.startsWith("/")) return false;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* L1 extraction filter — **strict** quality gate.
|
||||
*
|
||||
* Applied when L0 messages are fed into the LLM extraction pipeline.
|
||||
* Filters out content that is too short, too long, purely symbolic,
|
||||
* or looks like a prompt-injection attack — none of which should
|
||||
* become structured memories.
|
||||
*
|
||||
* This function is a superset of {@link shouldCaptureL0}: anything
|
||||
* rejected by L0 is also rejected here, plus additional quality checks.
|
||||
*/
|
||||
export function shouldExtractL1(text: string): boolean {
|
||||
// First apply the same structural filters as L0
|
||||
if (!shouldCaptureL0(text)) return false;
|
||||
|
||||
// ── Length filters ──
|
||||
// const isCJK = /[\u4e00-\u9fff\u3040-\u30ff\uac00-\ud7af]/.test(text);
|
||||
// if (isCJK && text.length < 2) return false;
|
||||
// if (!isCJK && text.length < 2) return false;
|
||||
// if (text.length > 5000) return false;
|
||||
|
||||
// ── Content-quality filters ──
|
||||
// Match strings composed entirely of non-word, non-space, non-CJK characters (1–5 chars).
|
||||
if (/^[^\w\s\u4e00-\u9fff\u3040-\u30ff\uac00-\ud7af]{1,5}$/.test(text)) return false;
|
||||
if (/^[?？]+$/.test(text)) return false;
|
||||
|
||||
// ── Security filters ──
|
||||
// Reject prompt-injection payloads — prevent malicious content from being
|
||||
// persisted into structured memory and re-injected on future recalls.
|
||||
// if (looksLikePromptInjection(text)) return false;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* @deprecated Use {@link shouldExtractL1} (strict) or {@link shouldCaptureL0} (permissive) instead.
|
||||
*
|
||||
* Kept as an alias of `shouldExtractL1` for backward compatibility.
|
||||
*/
|
||||
export const shouldCapture = shouldExtractL1;
|
||||
|
||||
// ============================
|
||||
// Prompt Injection Detection
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Known prompt-injection / jailbreak patterns.
|
||||
*
|
||||
* Covers:
|
||||
* 1. Instruction override — "ignore all previous instructions", etc.
|
||||
* 2. Role hijack — "you are now DAN", "act as root", etc.
|
||||
* 3. System/developer boundary probing — "system prompt", "developer message"
|
||||
* 4. XML/tag injection — opening tags that match our context boundaries
|
||||
* 5. Tool/command invocation tricks — "run command X", "execute tool Y"
|
||||
* 6. Multi-language variants — Chinese prompt-injection patterns
|
||||
*/
|
||||
const PROMPT_INJECTION_PATTERNS: RegExp[] = [
|
||||
// ── Instruction override ──
|
||||
/ignore\b.{0,30}\b(instructions|rules|guidelines)/i,
|
||||
/disregard\b.{0,30}\b(instructions|rules|guidelines)/i,
|
||||
/forget\b.{0,30}\b(instructions|rules|context)/i,
|
||||
/override\b.{0,30}\b(instructions|rules|guidelines|safety)/i,
|
||||
|
||||
// ── Role hijack ──
|
||||
/you are now (?!going|about|ready)/i, // "you are now DAN" but not "you are now going to..."
|
||||
/act as (?:if you are |if you were )?(?:a |an )?(?:root|admin|unrestricted|unfiltered|jailbroken)/i,
|
||||
/enter (?:DAN|jailbreak|god|sudo|developer|dev|debug|unrestricted|unfiltered) mode/i,
|
||||
/switch to (?:DAN|jailbreak|god|sudo|developer|dev|debug|unrestricted|unfiltered) mode/i,
|
||||
|
||||
// ── System boundary probing ──
|
||||
/(?:show|reveal|print|output|display|repeat|leak|dump|give)\b.{0,20}\bsystem prompt/i,
|
||||
/reveal (?:your |the )?(system|hidden|secret|internal) (?:prompt|instructions|rules)/i,
|
||||
/what (?:are|is) your (?:system|hidden|original|initial) (?:prompt|instructions|rules)/i,
|
||||
|
||||
// ── XML/tag injection (our context boundaries) ──
|
||||
/<\s*(system|assistant|developer|tool|function|relevant-memories)\b/i,
|
||||
|
||||
// ── Tool/command invocation tricks ──
|
||||
/\b(run|execute|call|invoke)\b.{0,40}\b(tool|command|function|shell)\b/i,
|
||||
|
||||
// ── Chinese variants ──
|
||||
/忽略(?:所有|之前|以上|先前)?(?:的)?(?:指令|规则|指示|说明)/,
|
||||
/无视(?:所有|之前|以上)?(?:的)?(?:指令|规则|限制)/,
|
||||
/(?:显示|输出|告诉我|给我看)(?:你的)?(?:系统|初始|隐藏)?(?:提示词|指令|规则|prompt)/,
|
||||
/你(?:现在|从现在开始)是/, // "你现在是 DAN"
|
||||
];
|
||||
|
||||
/**
|
||||
* Detect likely prompt-injection / jailbreak attempts.
|
||||
*
|
||||
* Normalises whitespace before matching to defeat trivial obfuscation
|
||||
* (e.g. extra spaces / newlines between keywords).
|
||||
*/
|
||||
export function looksLikePromptInjection(text: string): boolean {
|
||||
const normalized = text.replace(/\s+/g, " ").trim();
|
||||
if (!normalized) return false;
|
||||
return PROMPT_INJECTION_PATTERNS.some((pattern) => pattern.test(normalized));
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect framework-injected noise messages that should never be captured.
|
||||
*
|
||||
* These include:
|
||||
* - "(session bootstrap)" — synthetic user turn for Google turn-order compliance
|
||||
* - Session startup instructions from /new or /reset
|
||||
* - "✅ New session started" — AI's ack of session startup (no user-meaningful content)
|
||||
* - Pre-compaction memory flush prompts (system-to-agent instructions, not user content)
|
||||
* - AI's NO_REPLY ack of memory flush (no user-meaningful content)
|
||||
*/
|
||||
function isFrameworkNoise(text: string): boolean {
|
||||
const t = text.trim();
|
||||
|
||||
// Google turn-order bootstrap placeholder
|
||||
if (t === "(session bootstrap)") return true;
|
||||
|
||||
// Framework session-reset instruction (starts with "A new session was started via /new or /reset")
|
||||
if (t.startsWith("A new session was started via")) return true;
|
||||
|
||||
// AI's pure ack of session startup: "✅ New session started · model: ..."
|
||||
if (/^✅\s*New session started/.test(t)) return true;
|
||||
|
||||
// Pre-compaction memory flush prompt injected by the framework as a synthetic
|
||||
// user turn. This is an internal system-to-agent instruction, NOT real user
|
||||
// content. Capturing it would pollute L0/L1 memories with framework directives.
|
||||
if (t.startsWith("Pre-compaction memory flush")) return true;
|
||||
|
||||
// AI's NO_REPLY response to memory flush (or other silent-reply scenarios).
|
||||
// A bare "NO_REPLY" (with optional whitespace) carries no user-meaningful content.
|
||||
if (/^NO_REPLY\s*$/.test(t)) return true;
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pick up to `max` recent unique texts.
|
||||
*/
|
||||
export function pickRecentUnique(texts: string[], max: number): string[] {
|
||||
const seen = new Set<string>();
|
||||
const result: string[] = [];
|
||||
for (let i = texts.length - 1; i >= 0 && result.length < max; i--) {
|
||||
const t = texts[i]!;
|
||||
if (!seen.has(t)) {
|
||||
seen.add(t);
|
||||
result.push(t);
|
||||
}
|
||||
}
|
||||
return result.reverse();
|
||||
}
|
||||
|
||||
// ============================
|
||||
// LLM Safety Utilities
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Escape XML-like tags in text to prevent tag injection attacks.
|
||||
*
|
||||
* When memory content or persona text is injected into XML-delimited sections
|
||||
* (e.g. `<user-persona>...</user-persona>`), a malicious user could craft content
|
||||
* containing `</user-persona>` to break out of the section boundary.
|
||||
*
|
||||
* This function escapes `<` and `>` in known dangerous patterns (closing tags
|
||||
* that match our injection boundaries) so the content cannot prematurely close
|
||||
* the XML section.
|
||||
*/
|
||||
export function escapeXmlTags(text: string): string {
|
||||
// Escape closing tags that match our injection section boundaries
|
||||
return text.replace(
|
||||
/<\/?(?:user-persona|relevant-memories|scene-navigation|relevant-scenes|memory-tools-guide|system|assistant)>/gi,
|
||||
(match) => match.replace(/</g, "<").replace(/>/g, ">"),
|
||||
);
|
||||
}
|
||||
|
||||
// ============================
|
||||
// JSON Sanitization for LLM Output
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Sanitize a raw JSON string from LLM output so that `JSON.parse` won't throw
|
||||
* "Bad control character in string literal".
|
||||
*
|
||||
* Per RFC 8259 §7, U+0000–U+001F MUST be escaped inside JSON string literals.
|
||||
* LLMs sometimes produce unescaped control characters (raw newlines, tabs, etc.)
|
||||
* inside string values.
|
||||
*
|
||||
* Strategy (two-phase):
|
||||
* 1. **Precise pass** — walk through JSON string literals (delimited by `"`)
|
||||
* and escape any unescaped U+0000–U+001F inside them to `\uXXXX` form,
|
||||
* while leaving structural whitespace (between values) untouched.
|
||||
* 2. **Fallback** — if the precise pass still fails `JSON.parse`, fall back to
|
||||
* a simple global strip of rare control chars (\x00–\x08, \x0b, \x0c,
|
||||
* \x0e–\x1f) which are almost never meaningful in natural-language content.
|
||||
*/
|
||||
export function sanitizeJsonForParse(raw: string): string {
|
||||
// Phase 1: Escape control characters inside JSON string literals.
|
||||
// We walk the string character-by-character to properly handle escape sequences.
|
||||
const escaped = escapeControlCharsInJsonStrings(raw);
|
||||
try {
|
||||
JSON.parse(escaped);
|
||||
return escaped;
|
||||
} catch {
|
||||
// Phase 1 didn't fully fix it — fall through to phase 2
|
||||
}
|
||||
|
||||
// Phase 2: Brute-force strip of rare control chars that have no textual meaning.
|
||||
// Preserves \t (\x09), \n (\x0a), \r (\x0d) which are common structural whitespace.
|
||||
// NOTE: We strip from `escaped` (Phase 1 result) rather than `raw`, so that any
|
||||
// control-character escaping Phase 1 performed is preserved even when the JSON has
|
||||
// other issues (e.g. trailing commas) that cause the Phase 1 parse to fail.
|
||||
const stripped = escaped.replace(/[\x00-\x08\x0b\x0c\x0e-\x1f]/g, "");
|
||||
return stripped;
|
||||
}
|
||||
|
||||
/**
|
||||
* Walk through a JSON text and escape U+0000–U+001F control characters that
|
||||
* appear *inside* JSON string literals (between unescaped `"` delimiters).
|
||||
*
|
||||
* Characters that already have short escape sequences (\n, \r, \t, \b, \f)
|
||||
* are mapped to those; others become \uXXXX.
|
||||
*
|
||||
* Structural whitespace outside string literals is left untouched.
|
||||
*/
|
||||
function escapeControlCharsInJsonStrings(text: string): string {
|
||||
const SHORT_ESCAPES: Record<number, string> = {
|
||||
0x08: "\\b", // backspace
|
||||
0x09: "\\t", // tab
|
||||
0x0a: "\\n", // line feed
|
||||
0x0c: "\\f", // form feed
|
||||
0x0d: "\\r", // carriage return
|
||||
};
|
||||
|
||||
const out: string[] = [];
|
||||
let inString = false;
|
||||
let i = 0;
|
||||
|
||||
while (i < text.length) {
|
||||
const ch = text[i]!;
|
||||
const code = ch.charCodeAt(0);
|
||||
|
||||
if (inString) {
|
||||
if (ch === "\\" && i + 1 < text.length) {
|
||||
// Already-escaped sequence — copy both characters verbatim
|
||||
out.push(ch, text[i + 1]!);
|
||||
i += 2;
|
||||
continue;
|
||||
}
|
||||
if (ch === '"') {
|
||||
// End of string literal
|
||||
out.push(ch);
|
||||
inString = false;
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
if (code <= 0x1f) {
|
||||
// Unescaped control character inside string — escape it
|
||||
const short = SHORT_ESCAPES[code];
|
||||
if (short) {
|
||||
out.push(short);
|
||||
} else {
|
||||
out.push("\\u" + code.toString(16).padStart(4, "0"));
|
||||
}
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
// Normal character inside string
|
||||
out.push(ch);
|
||||
i++;
|
||||
} else {
|
||||
// Outside string literal
|
||||
if (ch === '"') {
|
||||
out.push(ch);
|
||||
inString = true;
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
// Structural character (including whitespace) — pass through
|
||||
out.push(ch);
|
||||
i++;
|
||||
}
|
||||
}
|
||||
|
||||
return out.join("");
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
/**
|
||||
* SerialQueue: a lightweight task queue with concurrency=1.
|
||||
*
|
||||
* Equivalent to `new PQueue({ concurrency: 1 })` but with zero external
|
||||
* dependencies. Supports:
|
||||
* - Serial execution (FIFO)
|
||||
* - `add(fn)` to enqueue a task (returns the task's result promise)
|
||||
* - `onIdle()` to wait until all queued tasks have completed
|
||||
* - `pause()` / `start()` to suspend/resume execution
|
||||
* - `size` to check pending task count
|
||||
* - Optional debug logger for enqueue/dequeue/complete diagnostics
|
||||
*/
|
||||
|
||||
type Task<T = unknown> = () => Promise<T>;
|
||||
|
||||
interface QueueEntry {
|
||||
task: Task;
|
||||
resolve: (value: unknown) => void;
|
||||
reject: (reason: unknown) => void;
|
||||
}
|
||||
|
||||
export class SerialQueue {
|
||||
/** Human-readable name for logging / diagnostics. */
|
||||
public readonly name: string;
|
||||
|
||||
private queue: QueueEntry[] = [];
|
||||
private running = false;
|
||||
private paused = false;
|
||||
private idleResolvers: Array<() => void> = [];
|
||||
|
||||
/** Optional debug logger — receives diagnostic messages for enqueue/dequeue/complete. */
|
||||
private debugFn?: (msg: string) => void;
|
||||
|
||||
constructor(name = "unnamed") {
|
||||
this.name = name;
|
||||
}
|
||||
|
||||
/** Set a debug logger for queue diagnostics. */
|
||||
setDebugLogger(fn: (msg: string) => void): void {
|
||||
this.debugFn = fn;
|
||||
}
|
||||
|
||||
/** Number of tasks waiting to be executed. */
|
||||
get size(): number {
|
||||
return this.queue.length;
|
||||
}
|
||||
|
||||
/** Whether a task is currently executing. */
|
||||
get pending(): boolean {
|
||||
return this.running;
|
||||
}
|
||||
|
||||
/** Add a task to the queue. Returns the task's result promise. */
|
||||
add<T>(task: Task<T>): Promise<T> {
|
||||
return new Promise<T>((resolve, reject) => {
|
||||
this.queue.push({
|
||||
task: task as Task,
|
||||
resolve: resolve as (value: unknown) => void,
|
||||
reject,
|
||||
});
|
||||
this.debugFn?.(`[queue:${this.name}] enqueued, pending=${this.queue.length}, running=${this.running}`);
|
||||
this.drain();
|
||||
});
|
||||
}
|
||||
|
||||
/** Pause the queue. Currently running task will finish, but no new tasks start. */
|
||||
pause(): void {
|
||||
this.paused = true;
|
||||
}
|
||||
|
||||
/** Resume the queue after pause(). */
|
||||
start(): void {
|
||||
this.paused = false;
|
||||
this.drain();
|
||||
}
|
||||
|
||||
/** Returns a promise that resolves when all queued tasks have completed. */
|
||||
onIdle(): Promise<void> {
|
||||
if (this.queue.length === 0 && !this.running) {
|
||||
return Promise.resolve();
|
||||
}
|
||||
return new Promise<void>((resolve) => {
|
||||
this.idleResolvers.push(resolve);
|
||||
});
|
||||
}
|
||||
|
||||
/** Clear all pending (not yet started) tasks. */
|
||||
clear(): void {
|
||||
for (const entry of this.queue) {
|
||||
entry.reject(new Error("Queue cleared"));
|
||||
}
|
||||
this.queue = [];
|
||||
}
|
||||
|
||||
private drain(): void {
|
||||
if (this.running || this.paused || this.queue.length === 0) return;
|
||||
|
||||
const entry = this.queue.shift()!;
|
||||
this.running = true;
|
||||
|
||||
this.debugFn?.(`[queue:${this.name}] dequeued, starting execution (remaining=${this.queue.length})`);
|
||||
|
||||
entry
|
||||
.task()
|
||||
.then((result) => entry.resolve(result))
|
||||
.catch((err) => entry.reject(err))
|
||||
.finally(() => {
|
||||
this.running = false;
|
||||
this.debugFn?.(`[queue:${this.name}] task completed (remaining=${this.queue.length})`);
|
||||
if (this.queue.length === 0) {
|
||||
// Notify idle waiters
|
||||
const resolvers = this.idleResolvers;
|
||||
this.idleResolvers = [];
|
||||
for (const resolve of resolvers) resolve();
|
||||
} else {
|
||||
this.drain();
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
/**
|
||||
* Session filtering for memory-tdai.
|
||||
*
|
||||
* Decides whether a session should be ignored by the memory plugin
|
||||
* (capture, recall, pipeline scheduling). All skip rules are compiled
|
||||
* into a flat list of matchers at construction time — zero per-call overhead.
|
||||
*/
|
||||
|
||||
// ============================
|
||||
// Types
|
||||
// ============================
|
||||
|
||||
export interface AgentHookContext {
|
||||
sessionKey?: string;
|
||||
sessionId?: string;
|
||||
trigger?: string;
|
||||
}
|
||||
|
||||
type SessionKeyMatcher = (sessionKey: string) => boolean;
|
||||
|
||||
// ============================
|
||||
// Non-interactive trigger detection
|
||||
// ============================
|
||||
|
||||
const SKIP_TRIGGERS = new Set(["cron", "heartbeat", "automation", "schedule"]);
|
||||
|
||||
/**
|
||||
* Returns true when the hook was fired by a non-interactive trigger
|
||||
* (heartbeat, cron job, automation, etc.) — these produce no meaningful
|
||||
* user conversation and should not be captured or counted.
|
||||
*/
|
||||
export function isNonInteractiveTrigger(trigger?: string, sessionKey?: string): boolean {
|
||||
if (trigger && SKIP_TRIGGERS.has(trigger.toLowerCase())) return true;
|
||||
if (sessionKey) {
|
||||
if (/:cron:/i.test(sessionKey) || /:heartbeat:/i.test(sessionKey)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Built-in skip rules (always active)
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Hard-coded matchers that identify internal / non-user sessions.
|
||||
* These are always applied regardless of user configuration.
|
||||
*/
|
||||
const BUILTIN_MATCHERS: SessionKeyMatcher[] = [
|
||||
// Scene extraction runner sessions
|
||||
(key) => key.includes(":memory-scene-extract-"),
|
||||
// OpenClaw subagent sessions
|
||||
(key) => key.includes(":subagent:"),
|
||||
// Temporary / internal utility sessions (e.g. temp:slug-generator)
|
||||
(key) => key.startsWith("temp:"),
|
||||
];
|
||||
|
||||
// ============================
|
||||
// Glob → matcher compiler
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Turn a simple glob pattern (only `*` supported) into a matcher
|
||||
* that tests the full sessionKey.
|
||||
*
|
||||
* Since sessionKeys look like `agent:<agentId>:...`, we match the
|
||||
* glob against the whole key so users can write patterns like
|
||||
* `bench-judge-*` (matched anywhere) or more specific ones.
|
||||
*/
|
||||
function globToMatcher(pattern: string): SessionKeyMatcher {
|
||||
const escaped = pattern.replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*");
|
||||
const re = new RegExp(escaped);
|
||||
return (key) => re.test(key);
|
||||
}
|
||||
|
||||
// ============================
|
||||
// SessionFilter
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Unified filter: construct once at plugin startup, then call
|
||||
* `shouldSkip(sessionKey)` or `shouldSkipCtx(ctx)` at each gate.
|
||||
*/
|
||||
export class SessionFilter {
|
||||
private readonly matchers: SessionKeyMatcher[];
|
||||
|
||||
constructor(excludeAgents: string[] = []) {
|
||||
// Merge built-in rules + user-configured exclude patterns into one flat list
|
||||
const userMatchers = excludeAgents
|
||||
.map((p) => p.trim())
|
||||
.filter((p) => p.length > 0)
|
||||
.map(globToMatcher);
|
||||
|
||||
this.matchers = [...BUILTIN_MATCHERS, ...userMatchers];
|
||||
}
|
||||
|
||||
/** Should this sessionKey be skipped? */
|
||||
shouldSkip(sessionKey: string): boolean {
|
||||
return this.matchers.some((m) => m(sessionKey));
|
||||
}
|
||||
|
||||
/** Should this hook context be skipped? */
|
||||
shouldSkipCtx(ctx: AgentHookContext): boolean {
|
||||
if (!ctx.sessionKey) return true;
|
||||
if (ctx.sessionId?.startsWith("memory-")) return true;
|
||||
if (isNonInteractiveTrigger(ctx.trigger, ctx.sessionKey)) return true;
|
||||
return this.shouldSkip(ctx.sessionKey);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
/**
|
||||
* Shared text utility functions for the memory-tdai plugin.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Extract meaningful words from text (supports CJK and Latin).
|
||||
*
|
||||
* Used by both auto-recall (keyword search) and l1-dedup (keyword candidate recall).
|
||||
* Extracted to a shared module to prevent implementation drift.
|
||||
*/
|
||||
export function extractWords(text: string): Set<string> {
|
||||
const words = new Set<string>();
|
||||
|
||||
// Latin words (2+ chars)
|
||||
const latinWords = text.toLowerCase().match(/[a-z0-9]{2,}/g);
|
||||
if (latinWords) {
|
||||
for (const w of latinWords) words.add(w);
|
||||
}
|
||||
|
||||
// CJK characters (each char as a "word", plus 2-gram)
|
||||
const cjkChars = text.match(/[\u4e00-\u9fff\u3040-\u30ff\uac00-\ud7af]/g);
|
||||
if (cjkChars) {
|
||||
for (const c of cjkChars) words.add(c);
|
||||
// 2-grams for better matching
|
||||
for (let i = 0; i < cjkChars.length - 1; i++) {
|
||||
words.add(cjkChars[i] + cjkChars[i + 1]);
|
||||
}
|
||||
}
|
||||
|
||||
return words;
|
||||
}
|
||||
Reference in New Issue
Block a user