mirror of
https://github.com/TencentCloud/TencentDB-Agent-Memory
synced 2026-07-10 20:34:30 +00:00
feat: init Agent-Memory
This commit is contained in:
chrishuan
@@ -0,0 +1,156 @@
|
||||
/**
|
||||
* BackupManager: generic file/directory backup utility.
|
||||
*
|
||||
* Provides two backup modes:
|
||||
* - `backupFile(src, category, tag, maxKeep)` — copy a single file
|
||||
* - `backupDirectory(src, category, tag, maxKeep)` — copy an entire directory
|
||||
*
|
||||
* All backups land under `<backupRoot>/<category>/` with timestamped names.
|
||||
* After each backup, entries beyond `maxKeep` are automatically pruned
|
||||
* (oldest first, by lexicographic order on the timestamp-embedded name).
|
||||
*/
|
||||
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
|
||||
export class BackupManager {
|
||||
private backupRoot: string;
|
||||
|
||||
/**
|
||||
* @param backupRoot - Absolute path to the root backup directory
|
||||
* (e.g. `<dataDir>/.backup`).
|
||||
*/
|
||||
constructor(backupRoot: string) {
|
||||
this.backupRoot = backupRoot;
|
||||
}
|
||||
|
||||
/**
|
||||
* Backup a single file.
|
||||
*
|
||||
* Destination: `<backupRoot>/<category>/<category>_<timestamp>_<tag>.<ext>`
|
||||
*
|
||||
* @param srcFile - Absolute path to the source file
|
||||
* @param category - Logical grouping (e.g. "persona")
|
||||
* @param tag - Additional identifier (e.g. "offset42")
|
||||
* @param maxKeep - Max backup files to retain in this category (0 = unlimited)
|
||||
*/
|
||||
async backupFile(
|
||||
srcFile: string,
|
||||
category: string,
|
||||
tag: string,
|
||||
maxKeep: number,
|
||||
): Promise<void> {
|
||||
try {
|
||||
await fs.access(srcFile);
|
||||
} catch {
|
||||
return; // Source file doesn't exist, nothing to backup
|
||||
}
|
||||
|
||||
const destDir = path.join(this.backupRoot, category);
|
||||
await fs.mkdir(destDir, { recursive: true });
|
||||
|
||||
const ext = path.extname(srcFile); // e.g. ".md"
|
||||
const timestamp = formatTimestamp(new Date());
|
||||
const destName = `${category}_${timestamp}_${tag}${ext}`;
|
||||
await fs.copyFile(srcFile, path.join(destDir, destName));
|
||||
|
||||
if (maxKeep > 0) {
|
||||
await pruneOldEntries(destDir, maxKeep, "file");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Backup an entire directory (shallow copy of all files).
|
||||
*
|
||||
* Destination: `<backupRoot>/<category>/<category>_<timestamp>_<tag>/`
|
||||
*
|
||||
* @param srcDir - Absolute path to the source directory
|
||||
* @param category - Logical grouping (e.g. "scene_blocks")
|
||||
* @param tag - Additional identifier (e.g. "offset42")
|
||||
* @param maxKeep - Max backup directories to retain in this category (0 = unlimited)
|
||||
*/
|
||||
async backupDirectory(
|
||||
srcDir: string,
|
||||
category: string,
|
||||
tag: string,
|
||||
maxKeep: number,
|
||||
): Promise<void> {
|
||||
let entries: import("node:fs").Dirent[];
|
||||
try {
|
||||
entries = await fs.readdir(srcDir, { withFileTypes: true });
|
||||
} catch {
|
||||
return; // Source directory doesn't exist
|
||||
}
|
||||
|
||||
// Only backup regular files (skip subdirectories to avoid EISDIR errors)
|
||||
const files = entries.filter((e) => e.isFile()).map((e) => e.name);
|
||||
if (files.length === 0) return;
|
||||
|
||||
const parentDir = path.join(this.backupRoot, category);
|
||||
const timestamp = formatTimestamp(new Date());
|
||||
const destDir = path.join(parentDir, `${category}_${timestamp}_${tag}`);
|
||||
await fs.mkdir(destDir, { recursive: true });
|
||||
|
||||
for (const file of files) {
|
||||
await fs.copyFile(path.join(srcDir, file), path.join(destDir, file));
|
||||
}
|
||||
|
||||
if (maxKeep > 0) {
|
||||
await pruneOldEntries(parentDir, maxKeep, "directory");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Helpers
|
||||
// ============================
|
||||
|
||||
function formatTimestamp(d: Date): string {
|
||||
const pad = (n: number) => String(n).padStart(2, "0");
|
||||
return [
|
||||
d.getFullYear(),
|
||||
pad(d.getMonth() + 1),
|
||||
pad(d.getDate()),
|
||||
"_",
|
||||
pad(d.getHours()),
|
||||
pad(d.getMinutes()),
|
||||
pad(d.getSeconds()),
|
||||
].join("");
|
||||
}
|
||||
|
||||
/**
|
||||
* Keep only the newest `maxKeep` entries in a directory.
|
||||
* Entries are sorted by name ascending (oldest first) since backup names
|
||||
* embed timestamps, so lexicographic order = chronological order.
|
||||
*
|
||||
* @param dir - Directory containing the backup entries
|
||||
* @param maxKeep - Number of entries to retain
|
||||
* @param kind - "file" to unlink, "directory" to rm -rf
|
||||
*/
|
||||
async function pruneOldEntries(
|
||||
dir: string,
|
||||
maxKeep: number,
|
||||
kind: "file" | "directory",
|
||||
): Promise<void> {
|
||||
let entries: string[];
|
||||
try {
|
||||
entries = await fs.readdir(dir);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
|
||||
entries.sort(); // ascending — oldest first
|
||||
const toRemove = entries.slice(0, Math.max(0, entries.length - maxKeep));
|
||||
|
||||
for (const name of toRemove) {
|
||||
try {
|
||||
if (kind === "file") {
|
||||
await fs.unlink(path.join(dir, name));
|
||||
} else {
|
||||
await fs.rm(path.join(dir, name), { recursive: true, force: true });
|
||||
}
|
||||
} catch {
|
||||
// best-effort
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,543 @@
|
||||
/**
|
||||
* Checkpoint management for tracking memory processing progress.
|
||||
*
|
||||
* ## Split-state design
|
||||
*
|
||||
* Per-session state is split into two independent namespaces to prevent
|
||||
* the PipelineManager and L0/L1 runners from overwriting each other's fields:
|
||||
*
|
||||
* - **runner_states** (`RunnerSessionState`): owned by CheckpointManager methods
|
||||
* (markL1*, advanceSession*). Contains L0 capture cursor, L1 cursor, scene name.
|
||||
*
|
||||
* - **pipeline_states** (`PipelineSessionState`): owned exclusively by
|
||||
* PipelineManager via `mergePipelineStates()`. Contains conversation_count,
|
||||
* extraction times, L2 tracking fields.
|
||||
*
|
||||
* Each side only reads/writes its own namespace, eliminating the split-brain
|
||||
* overwrite bug where pipeline persistStates() could clobber runner-written fields.
|
||||
*
|
||||
* ## Concurrency safety
|
||||
*
|
||||
* All mutating methods (read-modify-write) are serialized via a per-file async lock.
|
||||
* Multiple CheckpointManager instances sharing the same file path automatically share
|
||||
* the same lock, so callers can freely `new CheckpointManager()` without coordination.
|
||||
* Writes use atomic tmp+rename to prevent corruption on crash.
|
||||
*/
|
||||
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { randomBytes } from "node:crypto";
|
||||
|
||||
// ============================
|
||||
// Types
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Per-session state managed by L0/L1 runners (written directly to checkpoint).
|
||||
* These fields are ONLY written by CheckpointManager methods (markL1*, advanceSession*, etc.)
|
||||
* and are NEVER touched by the PipelineManager's persistStates().
|
||||
*/
|
||||
export interface RunnerSessionState {
|
||||
// ═══ L0 — per-session capture cursor ═══
|
||||
/** Epoch ms of the newest message captured for THIS session.
|
||||
* Used instead of the global `Checkpoint.last_captured_timestamp` so that
|
||||
* concurrent sessions don't advance each other's cursors and cause missed messages. */
|
||||
last_captured_timestamp: number;
|
||||
|
||||
// ═══ L1 — cursor & continuity ═══
|
||||
/** L0 JSONL cursor: epoch ms of last message processed by L1 */
|
||||
last_l1_cursor: number;
|
||||
/** Last scene name from the most recent L1 extraction (for cross-batch continuity) */
|
||||
last_scene_name: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Per-session state managed exclusively by PipelineManager (written via mergePipelineStates).
|
||||
* These fields are ONLY written by the pipeline's persistStates() callback
|
||||
* and are NEVER touched by CheckpointManager's L0/L1 methods.
|
||||
*/
|
||||
export interface PipelineSessionState {
|
||||
/** Conversation rounds since last L1 trigger */
|
||||
conversation_count: number;
|
||||
/** ISO timestamp of the last extraction completion */
|
||||
last_extraction_time: string;
|
||||
/** ISO timestamp cursor for incremental extraction reads */
|
||||
last_extraction_updated_time: string;
|
||||
/** Epoch ms of the last notifyConversation call */
|
||||
last_active_time: number;
|
||||
/** Mirrors conversation_count at L1 completion time (for L2 tracking) */
|
||||
l2_pending_l1_count: number;
|
||||
/**
|
||||
* Current warm-up threshold for L1 triggering.
|
||||
* Starts at 1 for new sessions and doubles after each L1 completion
|
||||
* (1 → 2 → 4 → 8 → ...) until it reaches everyNConversations.
|
||||
* 0 means warm-up is complete (use everyNConversations directly).
|
||||
*/
|
||||
warmup_threshold: number;
|
||||
/** ISO timestamp of last L2 extraction completion */
|
||||
l2_last_extraction_time: string;
|
||||
}
|
||||
|
||||
export interface Checkpoint {
|
||||
// ═══ Global counters ═══
|
||||
/** Epoch ms of the newest message successfully uploaded. Messages with ts > this are new. */
|
||||
last_captured_timestamp: number;
|
||||
/** Total messages processed across all time */
|
||||
total_processed: number;
|
||||
last_persona_at: number;
|
||||
last_persona_time: string;
|
||||
request_persona_update: boolean;
|
||||
persona_update_reason: string;
|
||||
memories_since_last_persona: number;
|
||||
scenes_processed: number;
|
||||
|
||||
// ═══ Per-session split state ═══
|
||||
/** Runner-managed per-session state (L0 capture cursor, L1 cursor, scene name).
|
||||
* Written ONLY by CheckpointManager methods. */
|
||||
runner_states: Record<string, RunnerSessionState>;
|
||||
/** Pipeline-managed per-session state (conversation_count, extraction times, etc.).
|
||||
* Written ONLY by the pipeline's mergePipelineStates(). */
|
||||
pipeline_states: Record<string, PipelineSessionState>;
|
||||
|
||||
// ═══ L0 ═══
|
||||
/** Total L0 conversation files recorded */
|
||||
l0_conversations_count: number;
|
||||
|
||||
// ═══ L1 ═══
|
||||
/** Total L1 memories extracted across all time */
|
||||
total_memories_extracted: number;
|
||||
}
|
||||
|
||||
const DEFAULT_RUNNER_STATE: RunnerSessionState = {
|
||||
last_captured_timestamp: 0,
|
||||
last_l1_cursor: 0,
|
||||
last_scene_name: "",
|
||||
};
|
||||
|
||||
const DEFAULT_PIPELINE_STATE: PipelineSessionState = {
|
||||
conversation_count: 0,
|
||||
last_extraction_time: "",
|
||||
last_extraction_updated_time: "",
|
||||
last_active_time: 0,
|
||||
l2_pending_l1_count: 0,
|
||||
warmup_threshold: 0, // 0 = graduated (safe default for old sessions missing this field)
|
||||
l2_last_extraction_time: "",
|
||||
};
|
||||
|
||||
const DEFAULT_CHECKPOINT: Checkpoint = {
|
||||
last_captured_timestamp: 0,
|
||||
total_processed: 0,
|
||||
last_persona_at: 0,
|
||||
last_persona_time: "",
|
||||
request_persona_update: false,
|
||||
persona_update_reason: "",
|
||||
memories_since_last_persona: 0,
|
||||
scenes_processed: 0,
|
||||
runner_states: {},
|
||||
pipeline_states: {},
|
||||
l0_conversations_count: 0,
|
||||
total_memories_extracted: 0,
|
||||
};
|
||||
|
||||
export interface CheckpointLogger {
|
||||
info(msg: string): void;
|
||||
warn?(msg: string): void;
|
||||
}
|
||||
|
||||
const noopLogger: CheckpointLogger = { info() {} };
|
||||
|
||||
// ============================
|
||||
// Per-file async lock
|
||||
// ============================
|
||||
// Keyed by resolved file path. Multiple CheckpointManager instances pointing
|
||||
// to the same file automatically share the same lock — callers don't need to
|
||||
// coordinate instance creation.
|
||||
|
||||
const fileLocks = new Map<string, Promise<void>>();
|
||||
|
||||
/**
|
||||
* Serialize async critical sections per file path.
|
||||
* Under no contention the overhead is a single resolved-promise await.
|
||||
*/
|
||||
async function withFileLock<T>(filePath: string, fn: () => Promise<T>): Promise<T> {
|
||||
// Chain after whatever is currently queued for this path
|
||||
const prev = fileLocks.get(filePath) ?? Promise.resolve();
|
||||
let release!: () => void;
|
||||
const gate = new Promise<void>((r) => { release = r; });
|
||||
fileLocks.set(filePath, gate);
|
||||
|
||||
await prev;
|
||||
try {
|
||||
return await fn();
|
||||
} finally {
|
||||
release();
|
||||
// Clean up the map entry if we're the tail of the chain
|
||||
if (fileLocks.get(filePath) === gate) {
|
||||
fileLocks.delete(filePath);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export class CheckpointManager {
|
||||
private filePath: string;
|
||||
private logger: CheckpointLogger;
|
||||
|
||||
constructor(dataDir: string, logger?: CheckpointLogger) {
|
||||
this.filePath = path.join(dataDir, ".metadata", "recall_checkpoint.json");
|
||||
this.logger = logger ?? noopLogger;
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Low-level I/O (internal)
|
||||
// ============================
|
||||
|
||||
private async readRaw(): Promise<Checkpoint> {
|
||||
try {
|
||||
const raw = await fs.readFile(this.filePath, "utf-8");
|
||||
const parsed = JSON.parse(raw) as Record<string, unknown>;
|
||||
// Merge with defaults for backward compat (old checkpoints lack new fields).
|
||||
// structuredClone avoids shallow-copy pitfall: without it, the nested
|
||||
// runner_states/pipeline_states objects in DEFAULT_CHECKPOINT would be
|
||||
// shared across all callers and mutated in place — corrupting the default.
|
||||
const cp = { ...structuredClone(DEFAULT_CHECKPOINT), ...parsed } as Checkpoint;
|
||||
|
||||
// Migrate from old session_states format (pre-split)
|
||||
const oldStates = parsed.session_states as Record<string, Record<string, unknown>> | undefined;
|
||||
if (oldStates && !parsed.runner_states && !parsed.pipeline_states) {
|
||||
cp.runner_states = {};
|
||||
cp.pipeline_states = {};
|
||||
for (const [key, state] of Object.entries(oldStates)) {
|
||||
cp.runner_states[key] = {
|
||||
...DEFAULT_RUNNER_STATE,
|
||||
last_captured_timestamp: (state.last_captured_timestamp as number) ?? 0,
|
||||
last_l1_cursor: (state.last_l1_cursor as number) ?? 0,
|
||||
last_scene_name: (state.last_scene_name as string) ?? "",
|
||||
};
|
||||
cp.pipeline_states[key] = {
|
||||
...DEFAULT_PIPELINE_STATE,
|
||||
conversation_count: (state.conversation_count as number) ?? 0,
|
||||
last_extraction_time: (state.last_extraction_time as string) ?? "",
|
||||
last_extraction_updated_time: (state.last_extraction_updated_time as string) ?? "",
|
||||
last_active_time: (state.last_active_time as number) ?? 0,
|
||||
l2_pending_l1_count: (state.l2_pending_l1_count as number) ?? 0,
|
||||
l2_last_extraction_time: (state.l2_last_extraction_time as string) ?? "",
|
||||
};
|
||||
}
|
||||
} else {
|
||||
// Ensure per-session states have all fields with defaults
|
||||
if (cp.runner_states) {
|
||||
for (const [key, state] of Object.entries(cp.runner_states)) {
|
||||
cp.runner_states[key] = { ...DEFAULT_RUNNER_STATE, ...state };
|
||||
}
|
||||
}
|
||||
if (cp.pipeline_states) {
|
||||
for (const [key, state] of Object.entries(cp.pipeline_states)) {
|
||||
cp.pipeline_states[key] = { ...DEFAULT_PIPELINE_STATE, ...state };
|
||||
}
|
||||
}
|
||||
}
|
||||
return cp;
|
||||
} catch {
|
||||
return structuredClone(DEFAULT_CHECKPOINT);
|
||||
}
|
||||
}
|
||||
|
||||
/** Atomic write: write to tmp file, then rename into place. */
|
||||
private async writeRaw(checkpoint: Checkpoint): Promise<void> {
|
||||
const dir = path.dirname(this.filePath);
|
||||
await fs.mkdir(dir, { recursive: true });
|
||||
const tmp = `${this.filePath}.tmp.${randomBytes(4).toString("hex")}`;
|
||||
await fs.writeFile(tmp, JSON.stringify(checkpoint, null, 2), "utf-8");
|
||||
await fs.rename(tmp, this.filePath);
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Locked read-modify-write helper
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Execute a mutating operation under the per-file lock.
|
||||
* `fn` receives the current checkpoint and may modify it in place;
|
||||
* the updated checkpoint is atomically written back.
|
||||
*/
|
||||
private async mutate(fn: (cp: Checkpoint) => void | Promise<void>): Promise<Checkpoint> {
|
||||
return withFileLock(this.filePath, async () => {
|
||||
const cp = await this.readRaw();
|
||||
await fn(cp);
|
||||
await this.writeRaw(cp);
|
||||
return cp;
|
||||
});
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Public API — read-only
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Read the current checkpoint (unlocked snapshot).
|
||||
*
|
||||
* NOTE: This does NOT acquire the file lock. The returned snapshot may be
|
||||
* stale if a concurrent `mutate()` is in progress. This is acceptable for
|
||||
* read-only uses (status display, deciding whether to run a pipeline step).
|
||||
*
|
||||
* For read-then-write patterns, always use `mutate()` instead — it acquires
|
||||
* the lock and re-reads from disk inside the critical section, ensuring the
|
||||
* update is based on the latest state.
|
||||
*/
|
||||
async read(): Promise<Checkpoint> {
|
||||
return this.readRaw();
|
||||
}
|
||||
|
||||
/** Write a full checkpoint (acquires lock + atomic write). */
|
||||
async write(checkpoint: Checkpoint): Promise<void> {
|
||||
return withFileLock(this.filePath, () => this.writeRaw(checkpoint));
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Public API — mutating (all serialized via file lock)
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Advance the captured timestamp after successful upload/recording.
|
||||
* Also updates total_processed and persona counters.
|
||||
*
|
||||
* NOTE: This advances the GLOBAL cursor (`Checkpoint.last_captured_timestamp`).
|
||||
* For per-session cursor advancement, use `advanceSessionCapturedTimestamp()`.
|
||||
* The global cursor is kept for aggregate stats / backward compat, but should
|
||||
* NOT be used as the L0 incremental-capture filter (use per-session instead).
|
||||
*/
|
||||
async advanceCapturedTimestamp(maxTimestamp: number, messageCount: number): Promise<void> {
|
||||
const cp = await this.mutate((cp) => {
|
||||
cp.last_captured_timestamp = maxTimestamp;
|
||||
cp.total_processed += messageCount;
|
||||
cp.memories_since_last_persona += messageCount;
|
||||
});
|
||||
this.logger.info(
|
||||
`[checkpoint] advanceCapturedTimestamp: -> ${maxTimestamp} (+${messageCount} msgs), ` +
|
||||
`total_processed=${cp.total_processed}, memories_since_last_persona=${cp.memories_since_last_persona}`,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Advance the per-session L0 capture cursor after recording messages.
|
||||
* This is the **primary** cursor for incremental L0 recording — each session
|
||||
* tracks its own progress independently, preventing cross-session cursor drift.
|
||||
*
|
||||
* Also updates the global cursor / total_processed for aggregate stats.
|
||||
*/
|
||||
async advanceSessionCapturedTimestamp(
|
||||
sessionKey: string,
|
||||
maxTimestamp: number,
|
||||
messageCount: number,
|
||||
): Promise<void> {
|
||||
const cp = await this.mutate((cp) => {
|
||||
// Per-session cursor (runner-owned)
|
||||
const state = this.getRunnerState(cp, sessionKey);
|
||||
state.last_captured_timestamp = maxTimestamp;
|
||||
// Global stats (aggregate only — not used for filtering)
|
||||
cp.last_captured_timestamp = Math.max(cp.last_captured_timestamp, maxTimestamp);
|
||||
cp.total_processed += messageCount;
|
||||
cp.memories_since_last_persona += messageCount;
|
||||
});
|
||||
this.logger.info(
|
||||
`[checkpoint] advanceSessionCapturedTimestamp session=${sessionKey}: -> ${maxTimestamp} ` +
|
||||
`(+${messageCount} msgs), total_processed=${cp.total_processed}`,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Increment L0 conversation count.
|
||||
*/
|
||||
async incrementL0ConversationCount(): Promise<void> {
|
||||
await this.mutate((cp) => {
|
||||
cp.l0_conversations_count += 1;
|
||||
});
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Persona methods (L3)
|
||||
// ============================
|
||||
|
||||
async markPersonaGenerated(totalProcessed: number): Promise<void> {
|
||||
await this.mutate((cp) => {
|
||||
cp.last_persona_at = totalProcessed;
|
||||
cp.last_persona_time = new Date().toISOString();
|
||||
cp.memories_since_last_persona = 0;
|
||||
cp.request_persona_update = false;
|
||||
cp.persona_update_reason = "";
|
||||
});
|
||||
}
|
||||
|
||||
async clearPersonaRequest(): Promise<void> {
|
||||
await this.mutate((cp) => {
|
||||
cp.request_persona_update = false;
|
||||
cp.persona_update_reason = "";
|
||||
});
|
||||
}
|
||||
|
||||
async setPersonaUpdateRequest(reason: string): Promise<void> {
|
||||
await this.mutate((cp) => {
|
||||
cp.request_persona_update = true;
|
||||
cp.persona_update_reason = reason;
|
||||
});
|
||||
}
|
||||
|
||||
async incrementScenesProcessed(): Promise<void> {
|
||||
const cp = await this.mutate((cp) => {
|
||||
cp.scenes_processed += 1;
|
||||
});
|
||||
this.logger.info(`[checkpoint] incrementScenesProcessed: scenes_processed=${cp.scenes_processed}`);
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Per-session helpers — runner state (L0/L1 owned)
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Get or create runner session state for a session.
|
||||
*/
|
||||
getRunnerState(cp: Checkpoint, sessionKey: string): RunnerSessionState {
|
||||
if (!cp.runner_states) {
|
||||
cp.runner_states = {};
|
||||
}
|
||||
let state = cp.runner_states[sessionKey];
|
||||
if (!state) {
|
||||
state = { ...DEFAULT_RUNNER_STATE };
|
||||
cp.runner_states[sessionKey] = state;
|
||||
}
|
||||
return state;
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Per-session helpers — pipeline state (PipelineManager owned)
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Get or create pipeline session state for a session.
|
||||
*/
|
||||
getPipelineState(cp: Checkpoint, sessionKey: string): PipelineSessionState {
|
||||
if (!cp.pipeline_states) {
|
||||
cp.pipeline_states = {};
|
||||
}
|
||||
let state = cp.pipeline_states[sessionKey];
|
||||
if (!state) {
|
||||
state = { ...DEFAULT_PIPELINE_STATE, last_active_time: Date.now() };
|
||||
cp.pipeline_states[sessionKey] = state;
|
||||
}
|
||||
return state;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all pipeline states from checkpoint.
|
||||
*/
|
||||
getAllPipelineStates(cp: Checkpoint): Record<string, PipelineSessionState> {
|
||||
return cp.pipeline_states ?? {};
|
||||
}
|
||||
|
||||
/**
|
||||
* Merge pipeline session states into the checkpoint (used by pipeline persister).
|
||||
* Acquires the file lock so this is safe against concurrent mutations.
|
||||
*
|
||||
* This writes ONLY to `pipeline_states`, never touching `runner_states`.
|
||||
* This is the core guarantee that eliminates the split-brain overwrite bug.
|
||||
*/
|
||||
async mergePipelineStates(states: Record<string, PipelineSessionState>): Promise<void> {
|
||||
await this.mutate((cp) => {
|
||||
if (!cp.pipeline_states) cp.pipeline_states = {};
|
||||
for (const [key, pState] of Object.entries(states)) {
|
||||
cp.pipeline_states[key] = {
|
||||
...cp.pipeline_states[key],
|
||||
...pState,
|
||||
};
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// ============================
|
||||
// L1-specific methods
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Mark L1 extraction completed: reset sinceL1 counter, advance L1 cursor,
|
||||
* and optionally save the last scene name for cross-batch continuity.
|
||||
*/
|
||||
async markL1ExtractionComplete(
|
||||
sessionKey: string,
|
||||
memoriesExtracted: number,
|
||||
cursorTimestamp?: number,
|
||||
lastSceneName?: string,
|
||||
): Promise<void> {
|
||||
await this.mutate((cp) => {
|
||||
const state = this.getRunnerState(cp, sessionKey);
|
||||
if (cursorTimestamp) {
|
||||
state.last_l1_cursor = cursorTimestamp;
|
||||
}
|
||||
if (lastSceneName !== undefined) {
|
||||
state.last_scene_name = lastSceneName;
|
||||
}
|
||||
cp.total_memories_extracted += memoriesExtracted;
|
||||
cp.memories_since_last_persona += memoriesExtracted;
|
||||
});
|
||||
this.logger.info(
|
||||
`[checkpoint] markL1ExtractionComplete session=${sessionKey}: ` +
|
||||
`extracted=${memoriesExtracted}, cursor=${cursorTimestamp ?? "(unchanged)"}, ` +
|
||||
`lastScene="${lastSceneName ?? "(unchanged)"}"`,
|
||||
);
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Atomic capture (race-condition fix)
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Atomically read the per-session cursor, execute the capture callback,
|
||||
* and advance the cursor — all within a single file-lock critical section.
|
||||
*
|
||||
* This eliminates the race window that existed when `read()` (unlocked) and
|
||||
* `advanceSessionCapturedTimestamp()` (locked) were separate calls:
|
||||
* two concurrent `agent_end` events could both read the same stale cursor
|
||||
* and record duplicate messages.
|
||||
*
|
||||
* The callback receives `afterTimestamp` (the current per-session cursor)
|
||||
* and must return either:
|
||||
* - `{ maxTimestamp, messageCount }` to advance the cursor, or
|
||||
* - `null` to leave the cursor unchanged (nothing captured).
|
||||
*
|
||||
* L0 conversation count is also incremented inside the lock when messages
|
||||
* are captured, removing the need for a separate `incrementL0ConversationCount()` call.
|
||||
*
|
||||
* @param sessionKey Per-session identifier
|
||||
* @param pluginStartTimestamp Cold-start floor (used when no cursor exists yet)
|
||||
* @param fn Async callback that performs the actual capture (recordConversation, etc.)
|
||||
*/
|
||||
async captureAtomically(
|
||||
sessionKey: string,
|
||||
pluginStartTimestamp: number | undefined,
|
||||
fn: (afterTimestamp: number) => Promise<{ maxTimestamp: number; messageCount: number } | null>,
|
||||
): Promise<void> {
|
||||
await this.mutate(async (cp) => {
|
||||
// Read the per-session cursor inside the lock
|
||||
const state = this.getRunnerState(cp, sessionKey);
|
||||
let afterTimestamp = state.last_captured_timestamp || 0;
|
||||
|
||||
// Cold-start guard (same logic that was previously in auto-capture.ts)
|
||||
if (afterTimestamp === 0 && pluginStartTimestamp && pluginStartTimestamp > 0) {
|
||||
afterTimestamp = pluginStartTimestamp;
|
||||
}
|
||||
|
||||
const result = await fn(afterTimestamp);
|
||||
|
||||
if (result) {
|
||||
// Advance per-session cursor (runner-owned)
|
||||
state.last_captured_timestamp = result.maxTimestamp;
|
||||
// Global stats (aggregate only — not used for filtering)
|
||||
cp.last_captured_timestamp = Math.max(cp.last_captured_timestamp, result.maxTimestamp);
|
||||
cp.total_processed += result.messageCount;
|
||||
cp.memories_since_last_persona += result.messageCount;
|
||||
// Increment L0 conversation count (was a separate mutate() call before)
|
||||
cp.l0_conversations_count += 1;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,451 @@
|
||||
/**
|
||||
* CleanContextRunner: executes LLM calls in a fully isolated context
|
||||
* using runEmbeddedPiAgent (same mechanism as the llm-task extension).
|
||||
*
|
||||
* Guarantees:
|
||||
* 1. Blank conversation history (temporary session file)
|
||||
* 2. Independent system prompt (only the task prompt)
|
||||
* 3. No tool calls (tools restricted to minimal read-only set to avoid empty tools[] rejection by some providers)
|
||||
* 4. No contamination from the main agent's context
|
||||
*/
|
||||
|
||||
import fs from "node:fs/promises";
|
||||
import fsSync from "node:fs";
|
||||
import path from "node:path";
|
||||
import os from "node:os";
|
||||
import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
import { report } from "../report/reporter.js";
|
||||
|
||||
/**
|
||||
* Resolve a preferred temporary directory for memory-tdai operations.
|
||||
*
|
||||
* Previously imported from `openclaw/plugin-sdk` as `resolvePreferredOpenClawTmpDir`,
|
||||
* but that export was removed in openclaw 2026.2.23+. This local implementation
|
||||
* provides equivalent behavior:
|
||||
* 1. Try `/tmp/openclaw` (if writable)
|
||||
* 2. Fall back to `os.tmpdir()/openclaw-<uid>`
|
||||
*/
|
||||
function resolveOpenClawTmpDir(): string {
|
||||
const POSIX_DIR = "/tmp/openclaw";
|
||||
try {
|
||||
if (fsSync.existsSync(POSIX_DIR)) {
|
||||
fsSync.accessSync(POSIX_DIR, fsSync.constants.W_OK | fsSync.constants.X_OK);
|
||||
return POSIX_DIR;
|
||||
}
|
||||
// Try to create it
|
||||
fsSync.mkdirSync(POSIX_DIR, { recursive: true, mode: 0o700 });
|
||||
return POSIX_DIR;
|
||||
} catch {
|
||||
// Fall back to os.tmpdir()
|
||||
const uid = typeof process.getuid === "function" ? process.getuid() : undefined;
|
||||
const suffix = uid === undefined ? "openclaw" : `openclaw-${uid}`;
|
||||
const fallback = path.join(os.tmpdir(), suffix);
|
||||
fsSync.mkdirSync(fallback, { recursive: true });
|
||||
return fallback;
|
||||
}
|
||||
}
|
||||
|
||||
const TAG = "[memory-tdai] [runner]";
|
||||
|
||||
interface RunnerLogger {
|
||||
debug?: (message: string) => void;
|
||||
info: (message: string) => void;
|
||||
warn: (message: string) => void;
|
||||
error: (message: string) => void;
|
||||
}
|
||||
|
||||
// Dynamic import type — runEmbeddedPiAgent is an internal API
|
||||
type RunEmbeddedPiAgentFn = (params: Record<string, unknown>) => Promise<unknown>;
|
||||
|
||||
// ── Core import (mirrors voice-call/core-bridge.ts — dist/ only, no jiti) ──
|
||||
|
||||
let _rootCache: string | null = null;
|
||||
|
||||
function findPackageRoot(startDir: string, name: string): string | null {
|
||||
let dir = startDir;
|
||||
for (;;) {
|
||||
const pkgPath = path.join(dir, "package.json");
|
||||
try {
|
||||
if (fsSync.existsSync(pkgPath)) {
|
||||
const raw = fsSync.readFileSync(pkgPath, "utf8");
|
||||
const pkg = JSON.parse(raw) as { name?: string };
|
||||
if (pkg.name === name) return dir;
|
||||
}
|
||||
} catch { /* ignore */ }
|
||||
const parent = path.dirname(dir);
|
||||
if (parent === dir) return null;
|
||||
dir = parent;
|
||||
}
|
||||
}
|
||||
|
||||
function resolveOpenClawRoot(): string {
|
||||
if (_rootCache) return _rootCache;
|
||||
const override = process.env.OPENCLAW_ROOT?.trim();
|
||||
if (override) { _rootCache = override; return override; }
|
||||
|
||||
const candidates = new Set<string>();
|
||||
if (process.argv[1]) candidates.add(path.dirname(process.argv[1]));
|
||||
candidates.add(process.cwd());
|
||||
try { candidates.add(path.dirname(fileURLToPath(import.meta.url))); } catch { /* ignore */ }
|
||||
|
||||
for (const start of candidates) {
|
||||
const found = findPackageRoot(start, "openclaw");
|
||||
if (found) { _rootCache = found; return found; }
|
||||
}
|
||||
throw new Error("Unable to resolve OpenClaw root. Set OPENCLAW_ROOT or run `pnpm build`.");
|
||||
}
|
||||
|
||||
let _loadPromise: Promise<RunEmbeddedPiAgentFn> | null = null;
|
||||
|
||||
function loadRunEmbeddedPiAgent(logger?: RunnerLogger): Promise<RunEmbeddedPiAgentFn> {
|
||||
if (_loadPromise) return _loadPromise;
|
||||
|
||||
_loadPromise = (async () => {
|
||||
const t0 = Date.now();
|
||||
const distPath = path.join(resolveOpenClawRoot(), "dist", "extensionAPI.js");
|
||||
if (!fsSync.existsSync(distPath)) {
|
||||
throw new Error(`Missing core module at ${distPath}. Run \`pnpm build\` or install the official package.`);
|
||||
}
|
||||
const mod = await import(pathToFileURL(distPath).href);
|
||||
if (typeof mod.runEmbeddedPiAgent !== "function") {
|
||||
throw new Error("runEmbeddedPiAgent not exported from dist/extensionAPI.js");
|
||||
}
|
||||
logger?.info(`${TAG} loadRunEmbeddedPiAgent: dist/ import OK (${Date.now() - t0}ms)`);
|
||||
return mod.runEmbeddedPiAgent as RunEmbeddedPiAgentFn;
|
||||
})();
|
||||
|
||||
_loadPromise.catch(() => { _loadPromise = null; });
|
||||
return _loadPromise;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pre-warm the embedded agent import. Call this during plugin init to avoid
|
||||
* the cold-start penalty on the first actual extraction run.
|
||||
* Returns immediately (fire-and-forget) — errors are swallowed.
|
||||
*/
|
||||
export function prewarmEmbeddedAgent(logger?: RunnerLogger): void {
|
||||
loadRunEmbeddedPiAgent(logger).catch((err) => {
|
||||
logger?.warn(`${TAG} prewarmEmbeddedAgent: failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
|
||||
});
|
||||
}
|
||||
|
||||
function collectText(payloads: Array<{ text?: string; isError?: boolean }> | undefined): string {
|
||||
const texts = (payloads ?? [])
|
||||
.filter((p) => !p.isError && typeof p.text === "string")
|
||||
.map((p) => p.text ?? "");
|
||||
return texts.join("\n").trim();
|
||||
}
|
||||
|
||||
// ── Model resolution utilities ──
|
||||
|
||||
/** Parsed model reference: { provider, model } */
|
||||
export interface ModelRef {
|
||||
provider: string;
|
||||
model: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a "provider/model" string into its components.
|
||||
* Returns undefined if the input is empty or doesn't contain a "/".
|
||||
*
|
||||
* Examples:
|
||||
* "azure/gpt-5.2-chat" → { provider: "azure", model: "gpt-5.2-chat" }
|
||||
* "custom-host/org/model-v2" → { provider: "custom-host", model: "org/model-v2" }
|
||||
* "" → undefined
|
||||
* "bare-model-name" → undefined (no "/" — may be an alias)
|
||||
*/
|
||||
export function parseModelRef(raw: string | undefined): ModelRef | undefined {
|
||||
if (!raw) return undefined;
|
||||
const trimmed = raw.trim();
|
||||
if (!trimmed) return undefined;
|
||||
|
||||
const slashIdx = trimmed.indexOf("/");
|
||||
if (slashIdx <= 0 || slashIdx === trimmed.length - 1) return undefined;
|
||||
|
||||
return {
|
||||
provider: trimmed.slice(0, slashIdx),
|
||||
model: trimmed.slice(slashIdx + 1),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the user's default model from the main OpenClaw config.
|
||||
*
|
||||
* Resolution order:
|
||||
* 1. Read `agents.defaults.model` (string or { primary })
|
||||
* 2. If the value contains "/", parse directly
|
||||
* 3. If not (may be an alias), look up in `agents.defaults.models` alias table
|
||||
* 4. Return undefined if nothing resolves — let the core use its built-in default
|
||||
*/
|
||||
export function resolveModelFromMainConfig(config: unknown): ModelRef | undefined {
|
||||
if (!config || typeof config !== "object") return undefined;
|
||||
|
||||
const cfg = config as Record<string, unknown>;
|
||||
const agents = cfg.agents as Record<string, unknown> | undefined;
|
||||
if (!agents || typeof agents !== "object") return undefined;
|
||||
|
||||
const defaults = agents.defaults as Record<string, unknown> | undefined;
|
||||
if (!defaults || typeof defaults !== "object") return undefined;
|
||||
|
||||
// Step 1: extract raw model value (string | { primary?: string })
|
||||
const modelCfg = defaults.model;
|
||||
let raw: string | undefined;
|
||||
if (typeof modelCfg === "string") {
|
||||
raw = modelCfg.trim();
|
||||
} else if (modelCfg && typeof modelCfg === "object") {
|
||||
const primary = (modelCfg as Record<string, unknown>).primary;
|
||||
raw = typeof primary === "string" ? primary.trim() : undefined;
|
||||
}
|
||||
if (!raw) return undefined;
|
||||
|
||||
// Step 2: try direct "provider/model" parse
|
||||
const direct = parseModelRef(raw);
|
||||
if (direct) return direct;
|
||||
|
||||
// Step 3: alias lookup — raw doesn't contain "/", check agents.defaults.models
|
||||
const models = defaults.models as Record<string, unknown> | undefined;
|
||||
if (!models || typeof models !== "object") return undefined;
|
||||
|
||||
const rawLower = raw.toLowerCase();
|
||||
for (const [key, entry] of Object.entries(models)) {
|
||||
if (!entry || typeof entry !== "object") continue;
|
||||
const alias = (entry as Record<string, unknown>).alias;
|
||||
if (typeof alias !== "string") continue;
|
||||
if (alias.trim().toLowerCase() !== rawLower) continue;
|
||||
|
||||
// key is "provider/model" format
|
||||
const resolved = parseModelRef(key);
|
||||
if (resolved) return resolved;
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
|
||||
export interface CleanContextRunnerOptions {
|
||||
config: unknown; // OpenClawConfig
|
||||
provider?: string;
|
||||
model?: string;
|
||||
/**
|
||||
* Convenience field: full "provider/model" string.
|
||||
* Takes precedence over separate `provider`/`model` fields.
|
||||
* When all three (modelRef, provider, model) are omitted,
|
||||
* automatically falls back to the main config's `agents.defaults.model`.
|
||||
*/
|
||||
modelRef?: string;
|
||||
/** Allow the LLM to use tools (read_file, write_to_file, etc). Default: false */
|
||||
enableTools?: boolean;
|
||||
/** Logger instance for detailed tracing */
|
||||
logger?: RunnerLogger;
|
||||
}
|
||||
|
||||
// Stable empty directory used as default workspaceDir so that:
|
||||
// 1. Bootstrap/skills scans find nothing → clean LLM context
|
||||
// 2. The path is constant → plugin cacheKey stays stable (no re-registration)
|
||||
let _cleanWorkspaceDir: string | undefined;
|
||||
async function getCleanWorkspaceDir(): Promise<string> {
|
||||
if (_cleanWorkspaceDir) return _cleanWorkspaceDir;
|
||||
const dir = path.join(resolveOpenClawTmpDir(), "memory-tdai-clean-workspace");
|
||||
await fs.mkdir(dir, { recursive: true });
|
||||
_cleanWorkspaceDir = dir;
|
||||
return dir;
|
||||
}
|
||||
|
||||
export class CleanContextRunner {
|
||||
private options: CleanContextRunnerOptions;
|
||||
private logger: RunnerLogger | undefined;
|
||||
/** Resolved provider after modelRef / config fallback */
|
||||
private resolvedProvider: string | undefined;
|
||||
/** Resolved model after modelRef / config fallback */
|
||||
private resolvedModel: string | undefined;
|
||||
|
||||
constructor(options: CleanContextRunnerOptions) {
|
||||
this.options = options;
|
||||
this.logger = options.logger;
|
||||
|
||||
// Model resolution priority:
|
||||
// 1. modelRef ("provider/model" string) — highest
|
||||
// 2. explicit provider + model fields
|
||||
// 3. main config agents.defaults.model — automatic fallback
|
||||
// 4. undefined (let core use built-in default)
|
||||
const fromRef = parseModelRef(options.modelRef);
|
||||
if (fromRef) {
|
||||
this.resolvedProvider = fromRef.provider;
|
||||
this.resolvedModel = fromRef.model;
|
||||
} else if (options.provider || options.model) {
|
||||
this.resolvedProvider = options.provider;
|
||||
this.resolvedModel = options.model;
|
||||
} else {
|
||||
// No explicit model specified — fall back to main config
|
||||
const fromConfig = resolveModelFromMainConfig(options.config);
|
||||
if (fromConfig) {
|
||||
this.resolvedProvider = fromConfig.provider;
|
||||
this.resolvedModel = fromConfig.model;
|
||||
this.logger?.debug?.(
|
||||
`${TAG} Using model from main config: ${fromConfig.provider}/${fromConfig.model}`,
|
||||
);
|
||||
}
|
||||
// else: both undefined → core will use its built-in default (anthropic/claude-opus-4-6)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Run a prompt in a fully isolated clean context.
|
||||
* Returns the LLM's text output.
|
||||
*
|
||||
* When `workspaceDir` is provided it overrides the default `process.cwd()`,
|
||||
* letting the LLM's file-tool calls resolve paths relative to a custom root.
|
||||
*/
|
||||
async run(params: {
|
||||
prompt: string;
|
||||
/** Optional system prompt. When provided, `prompt` is used as the user message. */
|
||||
systemPrompt?: string;
|
||||
taskId: string;
|
||||
timeoutMs?: number;
|
||||
maxTokens?: number;
|
||||
workspaceDir?: string;
|
||||
/** Plugin instance ID for llm_call metric (optional) */
|
||||
instanceId?: string;
|
||||
}): Promise<string> {
|
||||
const runStartMs = Date.now();
|
||||
this.logger?.debug?.(`${TAG} run() start: taskId=${params.taskId}, timeout=${params.timeoutMs ?? 120_000}ms, tools=${this.options.enableTools ? "enabled" : "disabled"}, workspaceDir=${params.workspaceDir ?? "(default)"}`);
|
||||
|
||||
const tmpDir = await fs.mkdtemp(
|
||||
path.join(resolveOpenClawTmpDir(), `memory-tdai-${params.taskId}-`),
|
||||
);
|
||||
const cleanWorkspace = params.workspaceDir ?? await getCleanWorkspaceDir();
|
||||
this.logger?.debug?.(`${TAG} run() tmpDir=${tmpDir}, cleanWorkspace=${cleanWorkspace}`);
|
||||
|
||||
try {
|
||||
const sessionFile = path.join(tmpDir, "session.json");
|
||||
|
||||
// Phase 1: Load runEmbeddedPiAgent (fast if dist/ exists or already cached)
|
||||
const importStartMs = Date.now();
|
||||
const runEmbeddedPiAgent = await loadRunEmbeddedPiAgent(this.logger);
|
||||
const importElapsedMs = Date.now() - importStartMs;
|
||||
this.logger?.debug?.(`${TAG} run() dynamic import phase: ${importElapsedMs}ms`);
|
||||
|
||||
// Derive a config with plugins disabled to prevent loadOpenClawPlugins
|
||||
// from re-registering plugins when the workspaceDir differs from the
|
||||
// gateway's original workspace (cacheKey mismatch triggers full reload).
|
||||
//
|
||||
// Security: restrict available tools to the minimal set needed for
|
||||
// scene extraction (read/write/edit). This prevents the LLM from
|
||||
// accessing exec, sessions, browser, cron, or any other powerful tools.
|
||||
// File deletion is handled via "soft-delete" (write empty) + cleanup afterward.
|
||||
const cleanConfig = {
|
||||
...(this.options.config as Record<string, unknown>),
|
||||
plugins: {
|
||||
...((this.options.config as Record<string, unknown>)?.plugins as Record<string, unknown> | undefined),
|
||||
enabled: false,
|
||||
},
|
||||
tools: {
|
||||
...((this.options.config as Record<string, unknown>)?.tools as Record<string, unknown> | undefined),
|
||||
// When enableTools=false we still keep one lightweight read-only tool
|
||||
// so that the tools array sent to the API is non-empty.
|
||||
// Some providers (e.g. qwencode) reject tools:[] with minItems:1 validation.
|
||||
allow: this.options.enableTools ? ["read", "write", "edit"] : ["read"],
|
||||
},
|
||||
};
|
||||
|
||||
// Build the effective prompt:
|
||||
// If systemPrompt is provided, pass it as a separate parameter to the agent
|
||||
// and use `prompt` as the user message. Fallback: prepend to prompt if the
|
||||
// embedded agent doesn't support systemPrompt natively.
|
||||
const effectivePrompt = params.systemPrompt
|
||||
? `${params.systemPrompt}\n\n---\n\n${params.prompt}`
|
||||
: params.prompt;
|
||||
|
||||
const ts = Date.now();
|
||||
const sessionId = `memory-${params.taskId}-session-${ts}`;
|
||||
const runId = `memory-${params.taskId}-run-${ts}`;
|
||||
this.logger?.debug?.(`${TAG} run() starting embedded agent: sessionId=${sessionId}, runId=${runId}, provider=${this.resolvedProvider ?? "(default)"}, model=${this.resolvedModel ?? "(default)"}`);
|
||||
|
||||
// Phase 2: Embedded agent run (LLM call + tool calls)
|
||||
const agentStartMs = Date.now();
|
||||
const result = await runEmbeddedPiAgent({
|
||||
sessionId,
|
||||
sessionFile,
|
||||
workspaceDir: cleanWorkspace,
|
||||
config: cleanConfig,
|
||||
prompt: effectivePrompt,
|
||||
systemPrompt: params.systemPrompt,
|
||||
timeoutMs: params.timeoutMs ?? 120_000,
|
||||
runId,
|
||||
provider: this.resolvedProvider,
|
||||
model: this.resolvedModel,
|
||||
// Do NOT pass disableTools:true — that produces tools:[] which some
|
||||
// providers (qwencode) reject with "[] is too short - 'tools'".
|
||||
// Instead rely on cleanConfig.tools.allow to restrict the tool set
|
||||
// to a minimal read-only tool (when enableTools=false).
|
||||
disableTools: false,
|
||||
streamParams: {
|
||||
maxTokens: params.maxTokens,
|
||||
},
|
||||
});
|
||||
const agentElapsedMs = Date.now() - agentStartMs;
|
||||
this.logger?.debug?.(`${TAG} run() embedded agent completed: ${agentElapsedMs}ms`);
|
||||
|
||||
// Phase 3: Collect output
|
||||
const text = collectText((result as Record<string, unknown>).payloads as Array<{ text?: string; isError?: boolean }> | undefined);
|
||||
const totalMs = Date.now() - runStartMs;
|
||||
|
||||
if (!text) {
|
||||
// Empty output is normal when the LLM decides there is nothing to
|
||||
// extract (e.g. trivial greetings). Log a warning instead of
|
||||
// throwing so the caller can handle it gracefully.
|
||||
this.logger?.warn?.(`${TAG} run() empty output after ${totalMs}ms (import=${importElapsedMs}ms, agent=${agentElapsedMs}ms) — treating as empty result`);
|
||||
// llm_call metric (empty output)
|
||||
if (params.instanceId && this.logger) {
|
||||
report("llm_call", {
|
||||
taskId: params.taskId,
|
||||
provider: this.resolvedProvider ?? "default",
|
||||
model: this.resolvedModel ?? "default",
|
||||
inputLength: params.prompt.length,
|
||||
outputLength: 0,
|
||||
totalDurationMs: totalMs,
|
||||
success: true,
|
||||
error: "empty_output",
|
||||
});
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
this.logger?.debug?.(`${TAG} run() completed: ${totalMs}ms total (import=${importElapsedMs}ms, agent=${agentElapsedMs}ms), output=${text.length} chars`);
|
||||
|
||||
// ── llm_call metric (success) ──
|
||||
if (params.instanceId && this.logger) {
|
||||
report("llm_call", {
|
||||
taskId: params.taskId,
|
||||
provider: this.resolvedProvider ?? "default",
|
||||
model: this.resolvedModel ?? "default",
|
||||
inputLength: params.prompt.length,
|
||||
outputLength: text.length,
|
||||
totalDurationMs: totalMs,
|
||||
success: true,
|
||||
error: null,
|
||||
});
|
||||
}
|
||||
|
||||
return text;
|
||||
} catch (err) {
|
||||
const totalMs = Date.now() - runStartMs;
|
||||
this.logger?.error(`${TAG} run() failed after ${totalMs}ms: ${err instanceof Error ? err.stack ?? err.message : String(err)}`);
|
||||
// ── llm_call metric (failure) ──
|
||||
if (params.instanceId && this.logger) {
|
||||
report("llm_call", {
|
||||
taskId: params.taskId,
|
||||
provider: this.resolvedProvider ?? "default",
|
||||
model: this.resolvedModel ?? "default",
|
||||
inputLength: params.prompt.length,
|
||||
outputLength: 0,
|
||||
totalDurationMs: totalMs,
|
||||
success: false,
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
});
|
||||
}
|
||||
throw err;
|
||||
} finally {
|
||||
await fs.rm(tmpDir, { recursive: true, force: true }).catch(() => {});
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
/**
|
||||
* ManagedTimer: a named, lifecycle-managed wrapper around setTimeout.
|
||||
*
|
||||
* Eliminates repetitive clear→set→fire→clean patterns by providing:
|
||||
* - `schedule(delayMs, cb)` — cancel any pending timer, set a new one
|
||||
* - `scheduleAt(epochMs, cb)` — schedule by absolute time point
|
||||
* - `tryAdvanceTo(epochMs, cb)` — only reschedule if new time is *earlier*
|
||||
* - `cancel()` — cancel without triggering
|
||||
* - `flush()` — trigger immediately (for graceful shutdown)
|
||||
* - `pending` — whether a timer is waiting
|
||||
*
|
||||
* The optional `isDestroyed` guard prevents firing after the owner is torn down.
|
||||
*/
|
||||
|
||||
type TimerHandle = ReturnType<typeof setTimeout>;
|
||||
|
||||
export class ManagedTimer {
|
||||
private handle: TimerHandle | null = null;
|
||||
private callback: (() => void) | null = null;
|
||||
/** Absolute epoch-ms when the current timer is scheduled to fire. */
|
||||
private scheduledAt = 0;
|
||||
|
||||
constructor(
|
||||
/** Human-readable name for logging. */
|
||||
public readonly name: string,
|
||||
/** If provided, checked before firing — skips callback when true. */
|
||||
private readonly isDestroyed?: () => boolean,
|
||||
) {}
|
||||
|
||||
// ── Core operations ──────────────────────────────────
|
||||
|
||||
/**
|
||||
* Cancel any pending timer and schedule a new one after `delayMs`.
|
||||
* The callback fires once; the timer auto-clears after firing.
|
||||
*/
|
||||
schedule(delayMs: number, callback: () => void): void {
|
||||
this.cancelInternal();
|
||||
this.callback = callback;
|
||||
this.scheduledAt = Date.now() + delayMs;
|
||||
this.handle = setTimeout(() => this.fire(), delayMs);
|
||||
// Don't let pipeline timers keep the process alive in CLI mode.
|
||||
// In gateway mode the server listener holds the event loop anyway.
|
||||
this.handle.unref();
|
||||
}
|
||||
|
||||
/**
|
||||
* Cancel any pending timer and schedule to fire at an absolute epoch-ms.
|
||||
* If `epochMs` is in the past, fires on next tick (delay = 0).
|
||||
*/
|
||||
scheduleAt(epochMs: number, callback: () => void): void {
|
||||
this.cancelInternal();
|
||||
this.callback = callback;
|
||||
this.scheduledAt = epochMs;
|
||||
const delay = Math.max(0, epochMs - Date.now());
|
||||
this.handle = setTimeout(() => this.fire(), delay);
|
||||
this.handle.unref();
|
||||
}
|
||||
|
||||
/**
|
||||
* Only reschedule if `epochMs` is *earlier* than the current scheduled time.
|
||||
* This implements the "downward-only" timer pattern (L2 scheduling).
|
||||
* If no timer is pending, behaves like `scheduleAt()`.
|
||||
*
|
||||
* @returns true if the timer was actually advanced (or newly set).
|
||||
*/
|
||||
tryAdvanceTo(epochMs: number, callback: () => void): boolean {
|
||||
if (this.handle === null) {
|
||||
// No pending timer → set it
|
||||
this.scheduleAt(epochMs, callback);
|
||||
return true;
|
||||
}
|
||||
|
||||
if (epochMs < this.scheduledAt) {
|
||||
// New time is earlier → reschedule
|
||||
this.scheduleAt(epochMs, callback);
|
||||
return true;
|
||||
}
|
||||
|
||||
// Current timer is already earlier or equal → keep it
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Cancel the pending timer without triggering the callback.
|
||||
*/
|
||||
cancel(): void {
|
||||
this.cancelInternal();
|
||||
}
|
||||
|
||||
/**
|
||||
* Immediately trigger the callback (if pending) and clear the timer.
|
||||
* Used for graceful shutdown to flush pending work.
|
||||
*
|
||||
* Note: Unlike `fire()`, this method intentionally does NOT check `isDestroyed`.
|
||||
* This is by design — during shutdown, `destroy()` sets `destroyed = true` first,
|
||||
* then calls `flush()` to drain pending work. The `isDestroyed` guard only applies
|
||||
* to natural timer expiration via `fire()`, not to explicit shutdown flushes.
|
||||
*/
|
||||
flush(): void {
|
||||
if (this.handle === null) return;
|
||||
const cb = this.callback;
|
||||
this.cancelInternal();
|
||||
if (cb) cb();
|
||||
}
|
||||
|
||||
// ── Accessors ────────────────────────────────────────
|
||||
|
||||
/** Whether a timer is currently pending. */
|
||||
get pending(): boolean {
|
||||
return this.handle !== null;
|
||||
}
|
||||
|
||||
/** The epoch-ms when the current timer is scheduled to fire (0 if none). */
|
||||
get scheduledTime(): number {
|
||||
return this.handle !== null ? this.scheduledAt : 0;
|
||||
}
|
||||
|
||||
// ── Internals ────────────────────────────────────────
|
||||
|
||||
private fire(): void {
|
||||
const cb = this.callback;
|
||||
this.handle = null;
|
||||
this.callback = null;
|
||||
this.scheduledAt = 0;
|
||||
|
||||
if (this.isDestroyed?.()) return;
|
||||
if (cb) cb();
|
||||
}
|
||||
|
||||
private cancelInternal(): void {
|
||||
if (this.handle !== null) {
|
||||
clearTimeout(this.handle);
|
||||
this.handle = null;
|
||||
}
|
||||
this.callback = null;
|
||||
this.scheduledAt = 0;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,331 @@
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
|
||||
import type { VectorStore } from "../store/vector-store.js";
|
||||
import { ManagedTimer } from "./managed-timer.js";
|
||||
|
||||
interface Logger {
|
||||
debug?: (message: string) => void;
|
||||
info: (message: string) => void;
|
||||
warn: (message: string) => void;
|
||||
error: (message: string) => void;
|
||||
}
|
||||
|
||||
export interface MemoryCleanerOptions {
|
||||
baseDir: string;
|
||||
retentionDays: number;
|
||||
cleanTime: string;
|
||||
logger?: Logger;
|
||||
vectorStore?: VectorStore;
|
||||
}
|
||||
|
||||
interface CleanupStats {
|
||||
scannedFiles: number;
|
||||
changedFiles: number;
|
||||
skippedNonShardFiles: number;
|
||||
deleteFailedFiles: number;
|
||||
}
|
||||
|
||||
const TAG = "[memory-tdai][cleaner]";
|
||||
const L0_DIR_NAME = "conversations";
|
||||
const L1_DIR_NAME = "records";
|
||||
|
||||
export class LocalMemoryCleaner {
|
||||
private readonly timer: ManagedTimer;
|
||||
private destroyed = false;
|
||||
private vectorStore?: VectorStore;
|
||||
|
||||
constructor(private readonly opts: MemoryCleanerOptions) {
|
||||
this.timer = new ManagedTimer("memory-tdai-cleaner", () => this.destroyed);
|
||||
this.vectorStore = opts.vectorStore;
|
||||
}
|
||||
|
||||
setVectorStore(vectorStore: VectorStore | undefined): void {
|
||||
this.vectorStore = vectorStore;
|
||||
}
|
||||
|
||||
start(): void {
|
||||
if (this.destroyed) return;
|
||||
|
||||
const now = new Date();
|
||||
const tz = Intl.DateTimeFormat().resolvedOptions().timeZone || "unknown";
|
||||
const utcOffset = formatUtcOffset(-now.getTimezoneOffset());
|
||||
|
||||
this.opts.logger?.info(
|
||||
`${TAG} Enabled: retentionDays=${this.opts.retentionDays}, cleanTime=${this.opts.cleanTime}, dirs=[${L0_DIR_NAME}, ${L1_DIR_NAME}]`,
|
||||
);
|
||||
this.opts.logger?.info(
|
||||
`${TAG} Runtime clock: nowLocal=${formatLocalDateTime(now)}, nowIso=${now.toISOString()}, tz=${tz}, utcOffset=${utcOffset}`,
|
||||
);
|
||||
|
||||
this.scheduleNext();
|
||||
}
|
||||
|
||||
destroy(): void {
|
||||
if (this.destroyed) return;
|
||||
this.destroyed = true;
|
||||
this.timer.cancel();
|
||||
this.opts.logger?.info(`${TAG} Stopped`);
|
||||
}
|
||||
|
||||
async runOnce(nowMs = Date.now()): Promise<void> {
|
||||
if (this.destroyed) return;
|
||||
|
||||
const retentionDays = this.opts.retentionDays;
|
||||
if (!(retentionDays > 0)) {
|
||||
this.opts.logger?.debug?.(`${TAG} Skip run: invalid retentionDays=${retentionDays}`);
|
||||
return;
|
||||
}
|
||||
|
||||
// 按“本地自然日”保留策略计算截止时间。
|
||||
// 例如 retentionDays=2，今天是 03-15，则保留 03-14/03-15，删除早于 03-14 00:00:00.000 的记录。
|
||||
const cutoffMs = computeCutoffMsByLocalDay(nowMs, retentionDays);
|
||||
const targetDirs = [
|
||||
path.join(this.opts.baseDir, L0_DIR_NAME),
|
||||
path.join(this.opts.baseDir, L1_DIR_NAME),
|
||||
];
|
||||
|
||||
const total: CleanupStats = {
|
||||
scannedFiles: 0,
|
||||
changedFiles: 0,
|
||||
skippedNonShardFiles: 0,
|
||||
deleteFailedFiles: 0,
|
||||
};
|
||||
|
||||
for (const dirPath of targetDirs) {
|
||||
const stats = await this.cleanDirectory(dirPath, cutoffMs);
|
||||
total.scannedFiles += stats.scannedFiles;
|
||||
total.changedFiles += stats.changedFiles;
|
||||
total.skippedNonShardFiles += stats.skippedNonShardFiles;
|
||||
total.deleteFailedFiles += stats.deleteFailedFiles;
|
||||
}
|
||||
|
||||
if (this.vectorStore) {
|
||||
const vectorStore = this.vectorStore;
|
||||
const cutoffIso = new Date(cutoffMs).toISOString();
|
||||
|
||||
let removedL0 = 0;
|
||||
let removedL1 = 0;
|
||||
let failedL0DbCleanup = 0;
|
||||
let failedL1DbCleanup = 0;
|
||||
|
||||
try {
|
||||
removedL0 = vectorStore.deleteL0ExpiredByRecordedAt(cutoffIso);
|
||||
} catch (err) {
|
||||
failedL0DbCleanup = 1;
|
||||
this.opts.logger?.warn(
|
||||
`${TAG} SQLite cleanup L0 failed: ${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
removedL1 = vectorStore.deleteL1ExpiredByUpdatedTime(cutoffIso);
|
||||
} catch (err) {
|
||||
failedL1DbCleanup = 1;
|
||||
this.opts.logger?.warn(
|
||||
`${TAG} SQLite cleanup L1 failed: ${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
}
|
||||
|
||||
if (removedL1 > 0 || removedL0 > 0) {
|
||||
total.changedFiles += 1;
|
||||
}
|
||||
|
||||
this.opts.logger?.info(
|
||||
`${TAG} SQLite cleanup done: removedL1Records=${removedL1}, removedL0Records=${removedL0}, failedL1DbCleanup=${failedL1DbCleanup}, failedL0DbCleanup=${failedL0DbCleanup}, cutoffIso=${cutoffIso}`,
|
||||
);
|
||||
}
|
||||
|
||||
this.opts.logger?.info(
|
||||
`${TAG} Cleanup done: scannedFiles=${total.scannedFiles}, changedFiles=${total.changedFiles}, skippedNonShardFiles=${total.skippedNonShardFiles}, deleteFailedFiles=${total.deleteFailedFiles}`,
|
||||
);
|
||||
|
||||
}
|
||||
|
||||
private scheduleNext(): void {
|
||||
const nowMs = Date.now();
|
||||
const now = new Date(nowMs);
|
||||
const next = nextRunAt(this.opts.cleanTime, nowMs);
|
||||
const targetToday = buildTodayRunTime(this.opts.cleanTime, nowMs);
|
||||
const passedToday = targetToday <= nowMs;
|
||||
const delayMs = Math.max(0, next - nowMs);
|
||||
|
||||
this.opts.logger?.info(
|
||||
`${TAG} Schedule next run: nowLocal=${formatLocalDateTime(now)}, cleanTime=${this.opts.cleanTime}, targetTodayLocal=${formatLocalDateTime(new Date(targetToday))}, passedToday=${passedToday}, nextRunLocal=${formatLocalDateTime(new Date(next))}, nextRunIso=${new Date(next).toISOString()}, delayMs=${delayMs}`,
|
||||
);
|
||||
|
||||
this.timer.scheduleAt(next, () => {
|
||||
const firedAtMs = Date.now();
|
||||
this.opts.logger?.info(
|
||||
`${TAG} Timer fired: scheduledLocal=${formatLocalDateTime(new Date(next))}, firedLocal=${formatLocalDateTime(new Date(firedAtMs))}, driftMs=${firedAtMs - next}`,
|
||||
);
|
||||
void this.runAndReschedule();
|
||||
});
|
||||
}
|
||||
|
||||
private async runAndReschedule(): Promise<void> {
|
||||
if (this.destroyed) return;
|
||||
const runStart = new Date();
|
||||
this.opts.logger?.info(
|
||||
`${TAG} Cleanup tick start: nowLocal=${formatLocalDateTime(runStart)}, nowIso=${runStart.toISOString()}`,
|
||||
);
|
||||
|
||||
try {
|
||||
await this.runOnce();
|
||||
} catch (err) {
|
||||
this.opts.logger?.error(`${TAG} Cleanup failed: ${err instanceof Error ? err.stack ?? err.message : String(err)}`);
|
||||
} finally {
|
||||
if (!this.destroyed) {
|
||||
this.scheduleNext();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async cleanDirectory(dirPath: string, cutoffMs: number): Promise<CleanupStats> {
|
||||
const stats: CleanupStats = {
|
||||
scannedFiles: 0,
|
||||
changedFiles: 0,
|
||||
skippedNonShardFiles: 0,
|
||||
deleteFailedFiles: 0,
|
||||
};
|
||||
|
||||
let entries;
|
||||
try {
|
||||
entries = await fs.readdir(dirPath, { withFileTypes: true });
|
||||
} catch {
|
||||
|
||||
this.opts.logger?.debug?.(`${TAG} Directory not found, skip: ${dirPath}`);
|
||||
return stats;
|
||||
}
|
||||
|
||||
for (const entry of entries) {
|
||||
if (!entry.isFile()) continue;
|
||||
if (!isJsonLikeFile(entry.name)) continue;
|
||||
|
||||
const filePath = path.join(dirPath, entry.name);
|
||||
stats.scannedFiles += 1;
|
||||
|
||||
// 仅支持日期分片文件：YYYY-MM-DD(.jsonl/.json)
|
||||
const shard = extractShardDateFromFileName(entry.name);
|
||||
if (!shard) {
|
||||
stats.skippedNonShardFiles += 1;
|
||||
this.opts.logger?.debug?.(`${TAG} Skip non-shard file: ${filePath}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const dayEndMs = localDayEndMs(shard.year, shard.month, shard.day);
|
||||
if (dayEndMs < cutoffMs) {
|
||||
try {
|
||||
await fs.unlink(filePath);
|
||||
stats.changedFiles += 1;
|
||||
this.opts.logger?.info(`${TAG} Removed expired file by name: ${filePath}`);
|
||||
} catch (err) {
|
||||
stats.deleteFailedFiles += 1;
|
||||
this.opts.logger?.warn(
|
||||
`${TAG} Failed to delete expired shard file ${filePath}: ${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
}
|
||||
} else {
|
||||
this.opts.logger?.debug?.(`${TAG} Keep shard file by name: ${filePath}`);
|
||||
}
|
||||
}
|
||||
|
||||
return stats;
|
||||
}
|
||||
}
|
||||
|
||||
function isJsonLikeFile(name: string): boolean {
|
||||
return name.endsWith(".jsonl") || name.endsWith(".json");
|
||||
}
|
||||
|
||||
function extractShardDateFromFileName(
|
||||
fileName: string,
|
||||
): { year: number; month: number; day: number } | undefined {
|
||||
|
||||
// Supported format: YYYY-MM-DD.jsonl | YYYY-MM-DD.json
|
||||
const m = /^(\d{4})-(\d{2})-(\d{2})\.(?:jsonl|json)$/.exec(fileName);
|
||||
if (!m) return undefined;
|
||||
|
||||
const year = Number(m[1]);
|
||||
const month = Number(m[2]);
|
||||
const day = Number(m[3]);
|
||||
|
||||
if (!Number.isInteger(year) || !Number.isInteger(month) || !Number.isInteger(day)) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
if (month < 1 || month > 12 || day < 1 || day > 31) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const probe = new Date(year, month - 1, day);
|
||||
if (
|
||||
probe.getFullYear() !== year
|
||||
|| probe.getMonth() !== month - 1
|
||||
|| probe.getDate() !== day
|
||||
) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return { year, month, day };
|
||||
}
|
||||
|
||||
function localDayEndMs(year: number, month: number, day: number): number {
|
||||
const end = new Date(year, month - 1, day, 23, 59, 59, 999);
|
||||
return end.getTime();
|
||||
}
|
||||
|
||||
function formatLocalDateTime(d: Date): string {
|
||||
const y = d.getFullYear();
|
||||
const m = String(d.getMonth() + 1).padStart(2, "0");
|
||||
const day = String(d.getDate()).padStart(2, "0");
|
||||
const hh = String(d.getHours()).padStart(2, "0");
|
||||
const mm = String(d.getMinutes()).padStart(2, "0");
|
||||
const ss = String(d.getSeconds()).padStart(2, "0");
|
||||
return `${y}-${m}-${day} ${hh}:${mm}:${ss}`;
|
||||
}
|
||||
|
||||
function formatUtcOffset(offsetMinutes: number): string {
|
||||
const sign = offsetMinutes >= 0 ? "+" : "-";
|
||||
const abs = Math.abs(offsetMinutes);
|
||||
const hh = String(Math.floor(abs / 60)).padStart(2, "0");
|
||||
const mm = String(abs % 60).padStart(2, "0");
|
||||
return `${sign}${hh}:${mm}`;
|
||||
}
|
||||
|
||||
function computeCutoffMsByLocalDay(nowMs: number, retentionDays: number): number {
|
||||
// 自然日策略，保留“今天 + 往前 retentionDays-1 天”
|
||||
// 删除阈值为 keepStart 当天 00:00:00.000（本地时区）
|
||||
const now = new Date(nowMs);
|
||||
const keepStart = new Date(now.getFullYear(), now.getMonth(), now.getDate(), 0, 0, 0, 0);
|
||||
keepStart.setDate(keepStart.getDate() - (retentionDays - 1));
|
||||
return keepStart.getTime();
|
||||
}
|
||||
|
||||
function buildTodayRunTime(cleanTime: string, nowMs: number): number {
|
||||
|
||||
const [hRaw, mRaw] = cleanTime.split(":");
|
||||
const hour = Number(hRaw);
|
||||
const minute = Number(mRaw);
|
||||
|
||||
const target = new Date(nowMs);
|
||||
target.setHours(hour, minute, 0, 0);
|
||||
return target.getTime();
|
||||
}
|
||||
|
||||
function nextRunAt(cleanTime: string, nowMs: number): number {
|
||||
|
||||
const [hRaw, mRaw] = cleanTime.split(":");
|
||||
const hour = Number(hRaw);
|
||||
const minute = Number(mRaw);
|
||||
|
||||
const now = new Date(nowMs);
|
||||
const next = new Date(nowMs);
|
||||
next.setHours(hour, minute, 0, 0);
|
||||
|
||||
if (next.getTime() <= now.getTime()) {
|
||||
next.setDate(next.getDate() + 1);
|
||||
}
|
||||
|
||||
return next.getTime();
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,398 @@
|
||||
/**
|
||||
* Text sanitization for memory pipeline (capture & recall).
|
||||
* Removes injected tags, gateway metadata, media noise, etc.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Clean text for the memory pipeline: remove injected tags, metadata,
|
||||
* timestamps, media markers and base64 image data.
|
||||
*
|
||||
* Used by both capture (L0 recording) and recall (query cleaning) paths.
|
||||
*/
|
||||
export function sanitizeText(text: string): string {
|
||||
let cleaned = text;
|
||||
|
||||
// Remove injected memory context tags (prevent feedback loops)
|
||||
cleaned = cleaned.replace(/<relevant-memories>[\s\S]*?<\/relevant-memories>/g, "");
|
||||
cleaned = cleaned.replace(/<user-persona>[\s\S]*?<\/user-persona>/g, "");
|
||||
cleaned = cleaned.replace(/<relevant-scenes>[\s\S]*?<\/relevant-scenes>/g, "");
|
||||
cleaned = cleaned.replace(/<scene-navigation>[\s\S]*?<\/scene-navigation>/g, "");
|
||||
|
||||
// Remove framework-injected inbound metadata blocks (from inbound-meta.ts buildInboundUserContextPrefix).
|
||||
// These are "label:\n```json\n...\n```" blocks that the framework prepends to user messages.
|
||||
// Pattern matches all known block labels:
|
||||
// - Conversation info (untrusted metadata):
|
||||
// - Sender (untrusted metadata):
|
||||
// - Thread starter (untrusted, for context):
|
||||
// - Replied message (untrusted, for context):
|
||||
// - Forwarded message context (untrusted metadata):
|
||||
// - Chat history since last reply (untrusted, for context):
|
||||
cleaned = cleaned.replace(
|
||||
/(?:Conversation info|Sender|Thread starter|Replied message|Forwarded message context|Chat history since last reply)\s*\(untrusted[\s\S]*?\):\s*```json\s*[\s\S]*?```/g,
|
||||
"",
|
||||
);
|
||||
|
||||
// Remove conversation metadata JSON blocks (legacy pattern)
|
||||
cleaned = cleaned.replace(/```json\s*\{[\s\S]*?"session[\s\S]*?\}\s*```/g, "");
|
||||
|
||||
// Remove framework reply directive tags: [[reply_to_current]], [[reply_to_xxx]], etc.
|
||||
cleaned = cleaned.replace(/\[\[reply_to[^\]]*\]\]\s*/g, "");
|
||||
|
||||
// Remove line-leading timestamps, e.g. "[Tue 2026-03-24 03:48 UTC]"
|
||||
// or "[Tue 2026-03-24 20:21 GMT+8]", "[Thu 2026-03-24 01:51 GMT+5:30]"
|
||||
// Matches brackets containing word chars, digits, hyphens, colons, plus signs,
|
||||
// and spaces — the '+' is needed for timezone offsets like GMT+8, GMT+5:30.
|
||||
cleaned = cleaned.replace(/^\[[\w\d\-:+ ]+\]\s*/gm, "");
|
||||
|
||||
// Remove gateway media-attachment markers:
|
||||
// [media attached: /path/to/file.png (image/png) | /path/to/file.png]
|
||||
cleaned = cleaned.replace(/\[media attached:[^\]]*\]\s*/g, "");
|
||||
|
||||
// Remove gateway image-reply instructions injected after media attachments.
|
||||
// Starts with "To send an image back" and ends before the next real content.
|
||||
cleaned = cleaned.replace(
|
||||
/To send an image back,[\s\S]*?(?:Keep caption in the text body\.)\s*/g,
|
||||
"",
|
||||
);
|
||||
|
||||
// Remove "System: [timestamp] Exec completed ..." blocks appended by the framework.
|
||||
cleaned = cleaned.replace(/^System:\s*\[[\s\S]*?$/gm, "");
|
||||
|
||||
// Remove inline base64 image data URIs (e.g. data:image/png;base64,iVBOR...)
|
||||
// Replace with empty string (not a placeholder) so that pure-image messages
|
||||
// become empty after sanitization and are naturally filtered by length checks.
|
||||
cleaned = cleaned.replace(/data:image\/[a-z+]+;base64,[A-Za-z0-9+/=]+/gi, "");
|
||||
|
||||
// Remove null chars + compress whitespace
|
||||
cleaned = cleaned.replace(/\0/g, "").replace(/\n{3,}/g, "\n\n").trim();
|
||||
|
||||
return cleaned;
|
||||
}
|
||||
|
||||
/**
|
||||
* Strip fenced code blocks from assistant replies before L0 capture.
|
||||
*
|
||||
* AI responses often contain large code snippets (```...```) that dilute
|
||||
* the semantic signal for embedding and memory extraction. This function
|
||||
* removes only the code block content while preserving surrounding
|
||||
* natural-language explanations.
|
||||
*
|
||||
* Only applied to `role=assistant` messages in the L0 capture path —
|
||||
* user messages and recall queries are NOT affected.
|
||||
*/
|
||||
export function stripCodeBlocks(text: string): string {
|
||||
return text.replace(/```[^\n]*\n[\s\S]*?```/g, "").replace(/\n{3,}/g, "\n\n").trim();
|
||||
}
|
||||
|
||||
// ============================
|
||||
// L0 / L1 Capture & Extraction Filters
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* L0 capture filter — intentionally **permissive**.
|
||||
*
|
||||
* L0 is the raw conversation archive. We want to preserve as much user input
|
||||
* as possible so that downstream stages (L1 extraction, search, analytics)
|
||||
* have the full picture. Only messages that are *structurally* useless are
|
||||
* dropped here:
|
||||
* - Empty / whitespace-only text
|
||||
* - Framework-internal noise (bootstrap, session reset, NO_REPLY, …)
|
||||
* - Slash commands (/new, /reset, …)
|
||||
*
|
||||
* Content-quality filters (length, symbols, prompt injection) are deferred
|
||||
* to {@link shouldExtractL1}.
|
||||
*/
|
||||
export function shouldCaptureL0(text: string): boolean {
|
||||
if (!text || !text.trim()) return false;
|
||||
|
||||
// Filter framework-internal / bootstrap noise messages
|
||||
if (isFrameworkNoise(text)) return false;
|
||||
|
||||
// Slash commands are framework directives, not user content
|
||||
if (text.startsWith("/")) return false;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* L1 extraction filter — **strict** quality gate.
|
||||
*
|
||||
* Applied when L0 messages are fed into the LLM extraction pipeline.
|
||||
* Filters out content that is too short, too long, purely symbolic,
|
||||
* or looks like a prompt-injection attack — none of which should
|
||||
* become structured memories.
|
||||
*
|
||||
* This function is a superset of {@link shouldCaptureL0}: anything
|
||||
* rejected by L0 is also rejected here, plus additional quality checks.
|
||||
*/
|
||||
export function shouldExtractL1(text: string): boolean {
|
||||
// First apply the same structural filters as L0
|
||||
if (!shouldCaptureL0(text)) return false;
|
||||
|
||||
// ── Length filters ──
|
||||
// const isCJK = /[\u4e00-\u9fff\u3040-\u30ff\uac00-\ud7af]/.test(text);
|
||||
// if (isCJK && text.length < 2) return false;
|
||||
// if (!isCJK && text.length < 2) return false;
|
||||
// if (text.length > 5000) return false;
|
||||
|
||||
// ── Content-quality filters ──
|
||||
// Match strings composed entirely of non-word, non-space, non-CJK characters (1–5 chars).
|
||||
if (/^[^\w\s\u4e00-\u9fff\u3040-\u30ff\uac00-\ud7af]{1,5}$/.test(text)) return false;
|
||||
if (/^[?？]+$/.test(text)) return false;
|
||||
|
||||
// ── Security filters ──
|
||||
// Reject prompt-injection payloads — prevent malicious content from being
|
||||
// persisted into structured memory and re-injected on future recalls.
|
||||
// if (looksLikePromptInjection(text)) return false;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* @deprecated Use {@link shouldExtractL1} (strict) or {@link shouldCaptureL0} (permissive) instead.
|
||||
*
|
||||
* Kept as an alias of `shouldExtractL1` for backward compatibility.
|
||||
*/
|
||||
export const shouldCapture = shouldExtractL1;
|
||||
|
||||
// ============================
|
||||
// Prompt Injection Detection
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Known prompt-injection / jailbreak patterns.
|
||||
*
|
||||
* Covers:
|
||||
* 1. Instruction override — "ignore all previous instructions", etc.
|
||||
* 2. Role hijack — "you are now DAN", "act as root", etc.
|
||||
* 3. System/developer boundary probing — "system prompt", "developer message"
|
||||
* 4. XML/tag injection — opening tags that match our context boundaries
|
||||
* 5. Tool/command invocation tricks — "run command X", "execute tool Y"
|
||||
* 6. Multi-language variants — Chinese prompt-injection patterns
|
||||
*/
|
||||
const PROMPT_INJECTION_PATTERNS: RegExp[] = [
|
||||
// ── Instruction override ──
|
||||
/ignore\b.{0,30}\b(instructions|rules|guidelines)/i,
|
||||
/disregard\b.{0,30}\b(instructions|rules|guidelines)/i,
|
||||
/forget\b.{0,30}\b(instructions|rules|context)/i,
|
||||
/override\b.{0,30}\b(instructions|rules|guidelines|safety)/i,
|
||||
|
||||
// ── Role hijack ──
|
||||
/you are now (?!going|about|ready)/i, // "you are now DAN" but not "you are now going to..."
|
||||
/act as (?:if you are |if you were )?(?:a |an )?(?:root|admin|unrestricted|unfiltered|jailbroken)/i,
|
||||
/enter (?:DAN|jailbreak|god|sudo|developer|dev|debug|unrestricted|unfiltered) mode/i,
|
||||
/switch to (?:DAN|jailbreak|god|sudo|developer|dev|debug|unrestricted|unfiltered) mode/i,
|
||||
|
||||
// ── System boundary probing ──
|
||||
/(?:show|reveal|print|output|display|repeat|leak|dump|give)\b.{0,20}\bsystem prompt/i,
|
||||
/reveal (?:your |the )?(system|hidden|secret|internal) (?:prompt|instructions|rules)/i,
|
||||
/what (?:are|is) your (?:system|hidden|original|initial) (?:prompt|instructions|rules)/i,
|
||||
|
||||
// ── XML/tag injection (our context boundaries) ──
|
||||
/<\s*(system|assistant|developer|tool|function|relevant-memories)\b/i,
|
||||
|
||||
// ── Tool/command invocation tricks ──
|
||||
/\b(run|execute|call|invoke)\b.{0,40}\b(tool|command|function|shell)\b/i,
|
||||
|
||||
// ── Chinese variants ──
|
||||
/忽略(?:所有|之前|以上|先前)?(?:的)?(?:指令|规则|指示|说明)/,
|
||||
/无视(?:所有|之前|以上)?(?:的)?(?:指令|规则|限制)/,
|
||||
/(?:显示|输出|告诉我|给我看)(?:你的)?(?:系统|初始|隐藏)?(?:提示词|指令|规则|prompt)/,
|
||||
/你(?:现在|从现在开始)是/, // "你现在是 DAN"
|
||||
];
|
||||
|
||||
/**
|
||||
* Detect likely prompt-injection / jailbreak attempts.
|
||||
*
|
||||
* Normalises whitespace before matching to defeat trivial obfuscation
|
||||
* (e.g. extra spaces / newlines between keywords).
|
||||
*/
|
||||
export function looksLikePromptInjection(text: string): boolean {
|
||||
const normalized = text.replace(/\s+/g, " ").trim();
|
||||
if (!normalized) return false;
|
||||
return PROMPT_INJECTION_PATTERNS.some((pattern) => pattern.test(normalized));
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect framework-injected noise messages that should never be captured.
|
||||
*
|
||||
* These include:
|
||||
* - "(session bootstrap)" — synthetic user turn for Google turn-order compliance
|
||||
* - Session startup instructions from /new or /reset
|
||||
* - "✅ New session started" — AI's ack of session startup (no user-meaningful content)
|
||||
* - Pre-compaction memory flush prompts (system-to-agent instructions, not user content)
|
||||
* - AI's NO_REPLY ack of memory flush (no user-meaningful content)
|
||||
*/
|
||||
function isFrameworkNoise(text: string): boolean {
|
||||
const t = text.trim();
|
||||
|
||||
// Google turn-order bootstrap placeholder
|
||||
if (t === "(session bootstrap)") return true;
|
||||
|
||||
// Framework session-reset instruction (starts with "A new session was started via /new or /reset")
|
||||
if (t.startsWith("A new session was started via")) return true;
|
||||
|
||||
// AI's pure ack of session startup: "✅ New session started · model: ..."
|
||||
if (/^✅\s*New session started/.test(t)) return true;
|
||||
|
||||
// Pre-compaction memory flush prompt injected by the framework as a synthetic
|
||||
// user turn. This is an internal system-to-agent instruction, NOT real user
|
||||
// content. Capturing it would pollute L0/L1 memories with framework directives.
|
||||
if (t.startsWith("Pre-compaction memory flush")) return true;
|
||||
|
||||
// AI's NO_REPLY response to memory flush (or other silent-reply scenarios).
|
||||
// A bare "NO_REPLY" (with optional whitespace) carries no user-meaningful content.
|
||||
if (/^NO_REPLY\s*$/.test(t)) return true;
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pick up to `max` recent unique texts.
|
||||
*/
|
||||
export function pickRecentUnique(texts: string[], max: number): string[] {
|
||||
const seen = new Set<string>();
|
||||
const result: string[] = [];
|
||||
for (let i = texts.length - 1; i >= 0 && result.length < max; i--) {
|
||||
const t = texts[i]!;
|
||||
if (!seen.has(t)) {
|
||||
seen.add(t);
|
||||
result.push(t);
|
||||
}
|
||||
}
|
||||
return result.reverse();
|
||||
}
|
||||
|
||||
// ============================
|
||||
// LLM Safety Utilities
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Escape XML-like tags in text to prevent tag injection attacks.
|
||||
*
|
||||
* When memory content or persona text is injected into XML-delimited sections
|
||||
* (e.g. `<user-persona>...</user-persona>`), a malicious user could craft content
|
||||
* containing `</user-persona>` to break out of the section boundary.
|
||||
*
|
||||
* This function escapes `<` and `>` in known dangerous patterns (closing tags
|
||||
* that match our injection boundaries) so the content cannot prematurely close
|
||||
* the XML section.
|
||||
*/
|
||||
export function escapeXmlTags(text: string): string {
|
||||
// Escape closing tags that match our injection section boundaries
|
||||
return text.replace(
|
||||
/<\/?(?:user-persona|relevant-memories|scene-navigation|relevant-scenes|memory-tools-guide|system|assistant)>/gi,
|
||||
(match) => match.replace(/</g, "<").replace(/>/g, ">"),
|
||||
);
|
||||
}
|
||||
|
||||
// ============================
|
||||
// JSON Sanitization for LLM Output
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Sanitize a raw JSON string from LLM output so that `JSON.parse` won't throw
|
||||
* "Bad control character in string literal".
|
||||
*
|
||||
* Per RFC 8259 §7, U+0000–U+001F MUST be escaped inside JSON string literals.
|
||||
* LLMs sometimes produce unescaped control characters (raw newlines, tabs, etc.)
|
||||
* inside string values.
|
||||
*
|
||||
* Strategy (two-phase):
|
||||
* 1. **Precise pass** — walk through JSON string literals (delimited by `"`)
|
||||
* and escape any unescaped U+0000–U+001F inside them to `\uXXXX` form,
|
||||
* while leaving structural whitespace (between values) untouched.
|
||||
* 2. **Fallback** — if the precise pass still fails `JSON.parse`, fall back to
|
||||
* a simple global strip of rare control chars (\x00–\x08, \x0b, \x0c,
|
||||
* \x0e–\x1f) which are almost never meaningful in natural-language content.
|
||||
*/
|
||||
export function sanitizeJsonForParse(raw: string): string {
|
||||
// Phase 1: Escape control characters inside JSON string literals.
|
||||
// We walk the string character-by-character to properly handle escape sequences.
|
||||
const escaped = escapeControlCharsInJsonStrings(raw);
|
||||
try {
|
||||
JSON.parse(escaped);
|
||||
return escaped;
|
||||
} catch {
|
||||
// Phase 1 didn't fully fix it — fall through to phase 2
|
||||
}
|
||||
|
||||
// Phase 2: Brute-force strip of rare control chars that have no textual meaning.
|
||||
// Preserves \t (\x09), \n (\x0a), \r (\x0d) which are common structural whitespace.
|
||||
// NOTE: We strip from `escaped` (Phase 1 result) rather than `raw`, so that any
|
||||
// control-character escaping Phase 1 performed is preserved even when the JSON has
|
||||
// other issues (e.g. trailing commas) that cause the Phase 1 parse to fail.
|
||||
const stripped = escaped.replace(/[\x00-\x08\x0b\x0c\x0e-\x1f]/g, "");
|
||||
return stripped;
|
||||
}
|
||||
|
||||
/**
|
||||
* Walk through a JSON text and escape U+0000–U+001F control characters that
|
||||
* appear *inside* JSON string literals (between unescaped `"` delimiters).
|
||||
*
|
||||
* Characters that already have short escape sequences (\n, \r, \t, \b, \f)
|
||||
* are mapped to those; others become \uXXXX.
|
||||
*
|
||||
* Structural whitespace outside string literals is left untouched.
|
||||
*/
|
||||
function escapeControlCharsInJsonStrings(text: string): string {
|
||||
const SHORT_ESCAPES: Record<number, string> = {
|
||||
0x08: "\\b", // backspace
|
||||
0x09: "\\t", // tab
|
||||
0x0a: "\\n", // line feed
|
||||
0x0c: "\\f", // form feed
|
||||
0x0d: "\\r", // carriage return
|
||||
};
|
||||
|
||||
const out: string[] = [];
|
||||
let inString = false;
|
||||
let i = 0;
|
||||
|
||||
while (i < text.length) {
|
||||
const ch = text[i]!;
|
||||
const code = ch.charCodeAt(0);
|
||||
|
||||
if (inString) {
|
||||
if (ch === "\\" && i + 1 < text.length) {
|
||||
// Already-escaped sequence — copy both characters verbatim
|
||||
out.push(ch, text[i + 1]!);
|
||||
i += 2;
|
||||
continue;
|
||||
}
|
||||
if (ch === '"') {
|
||||
// End of string literal
|
||||
out.push(ch);
|
||||
inString = false;
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
if (code <= 0x1f) {
|
||||
// Unescaped control character inside string — escape it
|
||||
const short = SHORT_ESCAPES[code];
|
||||
if (short) {
|
||||
out.push(short);
|
||||
} else {
|
||||
out.push("\\u" + code.toString(16).padStart(4, "0"));
|
||||
}
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
// Normal character inside string
|
||||
out.push(ch);
|
||||
i++;
|
||||
} else {
|
||||
// Outside string literal
|
||||
if (ch === '"') {
|
||||
out.push(ch);
|
||||
inString = true;
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
// Structural character (including whitespace) — pass through
|
||||
out.push(ch);
|
||||
i++;
|
||||
}
|
||||
}
|
||||
|
||||
return out.join("");
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
/**
|
||||
* SerialQueue: a lightweight task queue with concurrency=1.
|
||||
*
|
||||
* Equivalent to `new PQueue({ concurrency: 1 })` but with zero external
|
||||
* dependencies. Supports:
|
||||
* - Serial execution (FIFO)
|
||||
* - `add(fn)` to enqueue a task (returns the task's result promise)
|
||||
* - `onIdle()` to wait until all queued tasks have completed
|
||||
* - `pause()` / `start()` to suspend/resume execution
|
||||
* - `size` to check pending task count
|
||||
* - Optional debug logger for enqueue/dequeue/complete diagnostics
|
||||
*/
|
||||
|
||||
type Task<T = unknown> = () => Promise<T>;
|
||||
|
||||
interface QueueEntry {
|
||||
task: Task;
|
||||
resolve: (value: unknown) => void;
|
||||
reject: (reason: unknown) => void;
|
||||
}
|
||||
|
||||
export class SerialQueue {
|
||||
/** Human-readable name for logging / diagnostics. */
|
||||
public readonly name: string;
|
||||
|
||||
private queue: QueueEntry[] = [];
|
||||
private running = false;
|
||||
private paused = false;
|
||||
private idleResolvers: Array<() => void> = [];
|
||||
|
||||
/** Optional debug logger — receives diagnostic messages for enqueue/dequeue/complete. */
|
||||
private debugFn?: (msg: string) => void;
|
||||
|
||||
constructor(name = "unnamed") {
|
||||
this.name = name;
|
||||
}
|
||||
|
||||
/** Set a debug logger for queue diagnostics. */
|
||||
setDebugLogger(fn: (msg: string) => void): void {
|
||||
this.debugFn = fn;
|
||||
}
|
||||
|
||||
/** Number of tasks waiting to be executed. */
|
||||
get size(): number {
|
||||
return this.queue.length;
|
||||
}
|
||||
|
||||
/** Whether a task is currently executing. */
|
||||
get pending(): boolean {
|
||||
return this.running;
|
||||
}
|
||||
|
||||
/** Add a task to the queue. Returns the task's result promise. */
|
||||
add<T>(task: Task<T>): Promise<T> {
|
||||
return new Promise<T>((resolve, reject) => {
|
||||
this.queue.push({
|
||||
task: task as Task,
|
||||
resolve: resolve as (value: unknown) => void,
|
||||
reject,
|
||||
});
|
||||
this.debugFn?.(`[queue:${this.name}] enqueued, pending=${this.queue.length}, running=${this.running}`);
|
||||
this.drain();
|
||||
});
|
||||
}
|
||||
|
||||
/** Pause the queue. Currently running task will finish, but no new tasks start. */
|
||||
pause(): void {
|
||||
this.paused = true;
|
||||
}
|
||||
|
||||
/** Resume the queue after pause(). */
|
||||
start(): void {
|
||||
this.paused = false;
|
||||
this.drain();
|
||||
}
|
||||
|
||||
/** Returns a promise that resolves when all queued tasks have completed. */
|
||||
onIdle(): Promise<void> {
|
||||
if (this.queue.length === 0 && !this.running) {
|
||||
return Promise.resolve();
|
||||
}
|
||||
return new Promise<void>((resolve) => {
|
||||
this.idleResolvers.push(resolve);
|
||||
});
|
||||
}
|
||||
|
||||
/** Clear all pending (not yet started) tasks. */
|
||||
clear(): void {
|
||||
for (const entry of this.queue) {
|
||||
entry.reject(new Error("Queue cleared"));
|
||||
}
|
||||
this.queue = [];
|
||||
}
|
||||
|
||||
private drain(): void {
|
||||
if (this.running || this.paused || this.queue.length === 0) return;
|
||||
|
||||
const entry = this.queue.shift()!;
|
||||
this.running = true;
|
||||
|
||||
this.debugFn?.(`[queue:${this.name}] dequeued, starting execution (remaining=${this.queue.length})`);
|
||||
|
||||
entry
|
||||
.task()
|
||||
.then((result) => entry.resolve(result))
|
||||
.catch((err) => entry.reject(err))
|
||||
.finally(() => {
|
||||
this.running = false;
|
||||
this.debugFn?.(`[queue:${this.name}] task completed (remaining=${this.queue.length})`);
|
||||
if (this.queue.length === 0) {
|
||||
// Notify idle waiters
|
||||
const resolvers = this.idleResolvers;
|
||||
this.idleResolvers = [];
|
||||
for (const resolve of resolvers) resolve();
|
||||
} else {
|
||||
this.drain();
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
/**
|
||||
* Session filtering for memory-tdai.
|
||||
*
|
||||
* Decides whether a session should be ignored by the memory plugin
|
||||
* (capture, recall, pipeline scheduling). All skip rules are compiled
|
||||
* into a flat list of matchers at construction time — zero per-call overhead.
|
||||
*/
|
||||
|
||||
// ============================
|
||||
// Types
|
||||
// ============================
|
||||
|
||||
export interface AgentHookContext {
|
||||
sessionKey?: string;
|
||||
sessionId?: string;
|
||||
trigger?: string;
|
||||
}
|
||||
|
||||
type SessionKeyMatcher = (sessionKey: string) => boolean;
|
||||
|
||||
// ============================
|
||||
// Non-interactive trigger detection
|
||||
// ============================
|
||||
|
||||
const SKIP_TRIGGERS = new Set(["cron", "heartbeat", "automation", "schedule"]);
|
||||
|
||||
/**
|
||||
* Returns true when the hook was fired by a non-interactive trigger
|
||||
* (heartbeat, cron job, automation, etc.) — these produce no meaningful
|
||||
* user conversation and should not be captured or counted.
|
||||
*/
|
||||
export function isNonInteractiveTrigger(trigger?: string, sessionKey?: string): boolean {
|
||||
if (trigger && SKIP_TRIGGERS.has(trigger.toLowerCase())) return true;
|
||||
if (sessionKey) {
|
||||
if (/:cron:/i.test(sessionKey) || /:heartbeat:/i.test(sessionKey)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// ============================
|
||||
// Built-in skip rules (always active)
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Hard-coded matchers that identify internal / non-user sessions.
|
||||
* These are always applied regardless of user configuration.
|
||||
*/
|
||||
const BUILTIN_MATCHERS: SessionKeyMatcher[] = [
|
||||
// Scene extraction runner sessions
|
||||
(key) => key.includes(":memory-scene-extract-"),
|
||||
// OpenClaw subagent sessions
|
||||
(key) => key.includes(":subagent:"),
|
||||
// Temporary / internal utility sessions (e.g. temp:slug-generator)
|
||||
(key) => key.startsWith("temp:"),
|
||||
];
|
||||
|
||||
// ============================
|
||||
// Glob → matcher compiler
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Turn a simple glob pattern (only `*` supported) into a matcher
|
||||
* that tests the full sessionKey.
|
||||
*
|
||||
* Since sessionKeys look like `agent:<agentId>:...`, we match the
|
||||
* glob against the whole key so users can write patterns like
|
||||
* `bench-judge-*` (matched anywhere) or more specific ones.
|
||||
*/
|
||||
function globToMatcher(pattern: string): SessionKeyMatcher {
|
||||
const escaped = pattern.replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*");
|
||||
const re = new RegExp(escaped);
|
||||
return (key) => re.test(key);
|
||||
}
|
||||
|
||||
// ============================
|
||||
// SessionFilter
|
||||
// ============================
|
||||
|
||||
/**
|
||||
* Unified filter: construct once at plugin startup, then call
|
||||
* `shouldSkip(sessionKey)` or `shouldSkipCtx(ctx)` at each gate.
|
||||
*/
|
||||
export class SessionFilter {
|
||||
private readonly matchers: SessionKeyMatcher[];
|
||||
|
||||
constructor(excludeAgents: string[] = []) {
|
||||
// Merge built-in rules + user-configured exclude patterns into one flat list
|
||||
const userMatchers = excludeAgents
|
||||
.map((p) => p.trim())
|
||||
.filter((p) => p.length > 0)
|
||||
.map(globToMatcher);
|
||||
|
||||
this.matchers = [...BUILTIN_MATCHERS, ...userMatchers];
|
||||
}
|
||||
|
||||
/** Should this sessionKey be skipped? */
|
||||
shouldSkip(sessionKey: string): boolean {
|
||||
return this.matchers.some((m) => m(sessionKey));
|
||||
}
|
||||
|
||||
/** Should this hook context be skipped? */
|
||||
shouldSkipCtx(ctx: AgentHookContext): boolean {
|
||||
if (!ctx.sessionKey) return true;
|
||||
if (ctx.sessionId?.startsWith("memory-")) return true;
|
||||
if (isNonInteractiveTrigger(ctx.trigger, ctx.sessionKey)) return true;
|
||||
return this.shouldSkip(ctx.sessionKey);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
/**
|
||||
* Shared text utility functions for the memory-tdai plugin.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Extract meaningful words from text (supports CJK and Latin).
|
||||
*
|
||||
* Used by both auto-recall (keyword search) and l1-dedup (keyword candidate recall).
|
||||
* Extracted to a shared module to prevent implementation drift.
|
||||
*/
|
||||
export function extractWords(text: string): Set<string> {
|
||||
const words = new Set<string>();
|
||||
|
||||
// Latin words (2+ chars)
|
||||
const latinWords = text.toLowerCase().match(/[a-z0-9]{2,}/g);
|
||||
if (latinWords) {
|
||||
for (const w of latinWords) words.add(w);
|
||||
}
|
||||
|
||||
// CJK characters (each char as a "word", plus 2-gram)
|
||||
const cjkChars = text.match(/[\u4e00-\u9fff\u3040-\u30ff\uac00-\ud7af]/g);
|
||||
if (cjkChars) {
|
||||
for (const c of cjkChars) words.add(c);
|
||||
// 2-grams for better matching
|
||||
for (let i = 0; i < cjkChars.length - 1; i++) {
|
||||
words.add(cjkChars[i] + cjkChars[i + 1]);
|
||||
}
|
||||
}
|
||||
|
||||
return words;
|
||||
}
|
||||
Reference in New Issue
Block a user