feat: release v0.2.2 — TCVDB backend, BM25 hybrid retrieval, pipeline refactor

src/utils/checkpoint.ts

@@ -297,63 +297,6 @@ export class CheckpointManager {
   // Public API — mutating (all serialized via file lock)
   // ============================
 
-  /**
-   * Advance the captured timestamp after successful upload/recording.
-   * Also updates total_processed and persona counters.
-   *
-   * NOTE: This advances the GLOBAL cursor (`Checkpoint.last_captured_timestamp`).
-   * For per-session cursor advancement, use `advanceSessionCapturedTimestamp()`.
-   * The global cursor is kept for aggregate stats / backward compat, but should
-   * NOT be used as the L0 incremental-capture filter (use per-session instead).
-   */
-  async advanceCapturedTimestamp(maxTimestamp: number, messageCount: number): Promise<void> {
-    const cp = await this.mutate((cp) => {
-      cp.last_captured_timestamp = maxTimestamp;
-      cp.total_processed += messageCount;
-      cp.memories_since_last_persona += messageCount;
-    });
-    this.logger.info(
-      `[checkpoint] advanceCapturedTimestamp: -> ${maxTimestamp} (+${messageCount} msgs), ` +
-      `total_processed=${cp.total_processed}, memories_since_last_persona=${cp.memories_since_last_persona}`,
-    );
-  }
-
-  /**
-   * Advance the per-session L0 capture cursor after recording messages.
-   * This is the **primary** cursor for incremental L0 recording — each session
-   * tracks its own progress independently, preventing cross-session cursor drift.
-   *
-   * Also updates the global cursor / total_processed for aggregate stats.
-   */
-  async advanceSessionCapturedTimestamp(
-    sessionKey: string,
-    maxTimestamp: number,
-    messageCount: number,
-  ): Promise<void> {
-    const cp = await this.mutate((cp) => {
-      // Per-session cursor (runner-owned)
-      const state = this.getRunnerState(cp, sessionKey);
-      state.last_captured_timestamp = maxTimestamp;
-      // Global stats (aggregate only — not used for filtering)
-      cp.last_captured_timestamp = Math.max(cp.last_captured_timestamp, maxTimestamp);
-      cp.total_processed += messageCount;
-      cp.memories_since_last_persona += messageCount;
-    });
-    this.logger.info(
-      `[checkpoint] advanceSessionCapturedTimestamp session=${sessionKey}: -> ${maxTimestamp} ` +
-      `(+${messageCount} msgs), total_processed=${cp.total_processed}`,
-    );
-  }
-
-  /**
-   * Increment L0 conversation count.
-   */
-  async incrementL0ConversationCount(): Promise<void> {
-    await this.mutate((cp) => {
-      cp.l0_conversations_count += 1;
-    });
-  }
-
   // ============================
   // Persona methods (L3)
   // ============================
@@ -460,17 +403,20 @@ export class CheckpointManager {
   /**
    * Mark L1 extraction completed: reset sinceL1 counter, advance L1 cursor,
    * and optionally save the last scene name for cross-batch continuity.
+   *
+   * @param cursorRecordedAtMs - The max recorded_at epoch ms of processed L0 messages.
+   *   This becomes the new `last_l1_cursor` value (recorded_at semantics, not conversation timestamp).
    */
   async markL1ExtractionComplete(
     sessionKey: string,
     memoriesExtracted: number,
-    cursorTimestamp?: number,
+    cursorRecordedAtMs?: number,
     lastSceneName?: string,
   ): Promise<void> {
     await this.mutate((cp) => {
       const state = this.getRunnerState(cp, sessionKey);
-      if (cursorTimestamp) {
-        state.last_l1_cursor = cursorTimestamp;
+      if (cursorRecordedAtMs) {
+        state.last_l1_cursor = cursorRecordedAtMs;
       }
       if (lastSceneName !== undefined) {
         state.last_scene_name = lastSceneName;
@@ -480,7 +426,7 @@ export class CheckpointManager {
     });
     this.logger.info(
       `[checkpoint] markL1ExtractionComplete session=${sessionKey}: ` +
-      `extracted=${memoriesExtracted}, cursor=${cursorTimestamp ?? "(unchanged)"}, ` +
+      `extracted=${memoriesExtracted}, cursor=${cursorRecordedAtMs ?? "(unchanged)"}, ` +
       `lastScene="${lastSceneName ?? "(unchanged)"}"`,
     );
   }
@@ -533,7 +479,6 @@ export class CheckpointManager {
         // Global stats (aggregate only — not used for filtering)
         cp.last_captured_timestamp = Math.max(cp.last_captured_timestamp, result.maxTimestamp);
         cp.total_processed += result.messageCount;
-        cp.memories_since_last_persona += result.messageCount;
         // Increment L0 conversation count (was a separate mutate() call before)
         cp.l0_conversations_count += 1;
       }

src/utils/clean-context-runner.ts

@@ -14,6 +14,7 @@ import fsSync from "node:fs";
 import path from "node:path";
 import os from "node:os";
 import { fileURLToPath, pathToFileURL } from "node:url";
+import type { OpenClawPluginApi } from "openclaw/plugin-sdk/core";
 import { report } from "../report/reporter.js";
 
 /**
@@ -55,7 +56,42 @@ interface RunnerLogger {
 }
 
 // Dynamic import type — runEmbeddedPiAgent is an internal API
-type RunEmbeddedPiAgentFn = (params: Record<string, unknown>) => Promise<unknown>;
+// Prefer the public plugin runtime signature so host-injected runtimes stay assignable.
+type RunEmbeddedPiAgentFn = OpenClawPluginApi["runtime"]["agent"]["runEmbeddedPiAgent"];
+
+export interface EmbeddedAgentRuntimeLike {
+  runEmbeddedPiAgent?: RunEmbeddedPiAgentFn;
+}
+
+let _preferredAgentRuntime: EmbeddedAgentRuntimeLike | undefined;
+
+export function setPreferredEmbeddedAgentRuntime(
+  agentRuntime: EmbeddedAgentRuntimeLike | undefined,
+): void {
+  _preferredAgentRuntime = agentRuntime;
+}
+
+function resolveInjectedRunEmbeddedPiAgent(
+  agentRuntime?: EmbeddedAgentRuntimeLike,
+): RunEmbeddedPiAgentFn | undefined {
+  const candidate =
+    agentRuntime?.runEmbeddedPiAgent ?? _preferredAgentRuntime?.runEmbeddedPiAgent;
+  return typeof candidate === "function" ? candidate : undefined;
+}
+
+async function resolveRunEmbeddedPiAgent(
+  agentRuntime: EmbeddedAgentRuntimeLike | undefined,
+  logger?: RunnerLogger,
+): Promise<RunEmbeddedPiAgentFn> {
+  const injected = resolveInjectedRunEmbeddedPiAgent(agentRuntime);
+  if (injected) {
+    logger?.debug?.(
+      `${TAG} resolveRunEmbeddedPiAgent: using injected runtime.agent.runEmbeddedPiAgent`,
+    );
+    return injected;
+  }
+  return loadRunEmbeddedPiAgent(logger);
+}
 
 // ── Core import (mirrors voice-call/core-bridge.ts — dist/ only, no jiti) ──
 
@@ -123,7 +159,17 @@ function loadRunEmbeddedPiAgent(logger?: RunnerLogger): Promise<RunEmbeddedPiAge
  * the cold-start penalty on the first actual extraction run.
  * Returns immediately (fire-and-forget) — errors are swallowed.
  */
-export function prewarmEmbeddedAgent(logger?: RunnerLogger): void {
+export function prewarmEmbeddedAgent(
+  logger?: RunnerLogger,
+  agentRuntime?: EmbeddedAgentRuntimeLike,
+): void {
+  if (resolveInjectedRunEmbeddedPiAgent(agentRuntime)) {
+    logger?.debug?.(
+      `${TAG} prewarmEmbeddedAgent: runtime capability already available, skipping legacy preload`,
+    );
+    return;
+  }
+
   loadRunEmbeddedPiAgent(logger).catch((err) => {
     logger?.warn(`${TAG} prewarmEmbeddedAgent: failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
   });
@@ -232,6 +278,8 @@ export interface CleanContextRunnerOptions {
    * automatically falls back to the main config's `agents.defaults.model`.
    */
   modelRef?: string;
+  /** Preferred runtime seam. When absent, falls back to the legacy dist bridge. */
+  agentRuntime?: EmbeddedAgentRuntimeLike;
   /** Allow the LLM to use tools (read_file, write_to_file, etc). Default: false */
   enableTools?: boolean;
   /** Logger instance for detailed tracing */
@@ -318,11 +366,14 @@ export class CleanContextRunner {
     try {
       const sessionFile = path.join(tmpDir, "session.json");
 
-      // Phase 1: Load runEmbeddedPiAgent (fast if dist/ exists or already cached)
+      // Phase 1: Resolve runEmbeddedPiAgent (prefer runtime, fallback to legacy dist bridge)
       const importStartMs = Date.now();
-      const runEmbeddedPiAgent = await loadRunEmbeddedPiAgent(this.logger);
+      const runEmbeddedPiAgent = await resolveRunEmbeddedPiAgent(
+        this.options.agentRuntime,
+        this.logger,
+      );
       const importElapsedMs = Date.now() - importStartMs;
-      this.logger?.debug?.(`${TAG} run() dynamic import phase: ${importElapsedMs}ms`);
+      this.logger?.debug?.(`${TAG} run() runner resolution phase: ${importElapsedMs}ms`);
 
       // Derive a config with plugins disabled to prevent loadOpenClawPlugins
       // from re-registering plugins when the workspaceDir differs from the
@@ -347,10 +398,10 @@ export class CleanContextRunner {
         },
       };
 
-      // Build the effective prompt:
-      // If systemPrompt is provided, pass it as a separate parameter to the agent
-      // and use `prompt` as the user message. Fallback: prepend to prompt if the
-      // embedded agent doesn't support systemPrompt natively.
+      // Build the effective prompt.
+      // Keep prepending the optional systemPrompt into the user-visible prompt so
+      // runtime and legacy fallback paths preserve the same behavior without
+      // relying on a newer native extraSystemPrompt contract.
       const effectivePrompt = params.systemPrompt
         ? `${params.systemPrompt}\n\n---\n\n${params.prompt}`
         : params.prompt;
@@ -368,7 +419,6 @@ export class CleanContextRunner {
         workspaceDir: cleanWorkspace,
         config: cleanConfig,
         prompt: effectivePrompt,
-        systemPrompt: params.systemPrompt,
         timeoutMs: params.timeoutMs ?? 120_000,
         runId,
         provider: this.resolvedProvider,

src/utils/manifest.ts

@@ -0,0 +1,159 @@
+/**
+ * Manifest — self-describing metadata for a memory-tdai data directory.
+ *
+ * Lives at `<dataDir>/.metadata/manifest.json`.
+ *
+ * - **store**: written once on first successful store init; never overwritten.
+ *   On subsequent starts the current config is compared against the persisted
+ *   store binding — mismatches are logged as warnings.
+ * - **seed**: written once when a seed run completes; null for live-runtime dirs.
+ *
+ * This file is informational / read-only from the user's perspective.
+ * The plugin reads it on startup for consistency checks.
+ */
+
+import fs from "node:fs";
+import path from "node:path";
+
+// ============================
+// Types
+// ============================
+
+export interface ManifestStoreInfo {
+  type: "sqlite" | "tcvdb";
+  sqlite?: {
+    /** Relative path to the SQLite DB file (relative to dataDir). */
+    path: string;
+  };
+  tcvdb?: {
+    url: string;
+    database: string;
+    /** User-friendly alias (optional). */
+    alias?: string;
+  };
+}
+
+export interface ManifestSeedInfo {
+  /** Original input file name (basename only). */
+  inputFile?: string;
+  sessions: number;
+  rounds: number;
+  messages: number;
+  startedAt: string;
+  completedAt: string;
+}
+
+export interface Manifest {
+  /** Schema version for future migrations. */
+  version: 1;
+  /** Timestamp when the manifest was first created. */
+  createdAt: string;
+  /** Store binding — written once on first init. */
+  store: ManifestStoreInfo;
+  /** Seed run info — null for live-runtime directories. */
+  seed: ManifestSeedInfo | null;
+}
+
+// ============================
+// Paths
+// ============================
+
+const METADATA_DIR = ".metadata";
+const MANIFEST_FILE = "manifest.json";
+
+export function manifestPath(dataDir: string): string {
+  return path.join(dataDir, METADATA_DIR, MANIFEST_FILE);
+}
+
+// ============================
+// Read / Write
+// ============================
+
+/**
+ * Read an existing manifest from disk. Returns `null` if not found or unparseable.
+ */
+export function readManifest(dataDir: string): Manifest | null {
+  const p = manifestPath(dataDir);
+  try {
+    if (!fs.existsSync(p)) return null;
+    const raw = fs.readFileSync(p, "utf-8");
+    return JSON.parse(raw) as Manifest;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Write a manifest to disk (creates `.metadata/` if needed).
+ */
+export function writeManifest(dataDir: string, manifest: Manifest): void {
+  const dir = path.join(dataDir, METADATA_DIR);
+  fs.mkdirSync(dir, { recursive: true });
+  fs.writeFileSync(
+    manifestPath(dataDir),
+    JSON.stringify(manifest, null, 2) + "\n",
+    "utf-8",
+  );
+}
+
+// ============================
+// Store binding helpers
+// ============================
+
+export interface StoreConfigSnapshot {
+  type: "sqlite" | "tcvdb";
+  sqlitePath?: string;
+  tcvdbUrl?: string;
+  tcvdbDatabase?: string;
+  tcvdbAlias?: string;
+}
+
+/**
+ * Build a ManifestStoreInfo from the current store config snapshot.
+ */
+export function buildStoreInfo(snapshot: StoreConfigSnapshot): ManifestStoreInfo {
+  const info: ManifestStoreInfo = { type: snapshot.type };
+  if (snapshot.type === "sqlite") {
+    info.sqlite = { path: snapshot.sqlitePath ?? "vectors.db" };
+  } else {
+    info.tcvdb = {
+      url: snapshot.tcvdbUrl!,
+      database: snapshot.tcvdbDatabase!,
+      alias: snapshot.tcvdbAlias || undefined,
+    };
+  }
+  return info;
+}
+
+/**
+ * Compare the persisted store binding against the current config.
+ * Returns a list of human-readable mismatch descriptions (empty = all good).
+ */
+export function diffStoreBinding(
+  persisted: ManifestStoreInfo,
+  current: ManifestStoreInfo,
+): string[] {
+  const diffs: string[] = [];
+
+  if (persisted.type !== current.type) {
+    diffs.push(`store type changed: ${persisted.type} → ${current.type}`);
+    return diffs; // no point comparing fields across different types
+  }
+
+  if (persisted.type === "sqlite" && current.type === "sqlite") {
+    if (persisted.sqlite?.path !== current.sqlite?.path) {
+      diffs.push(`sqlite path changed: ${persisted.sqlite?.path} → ${current.sqlite?.path}`);
+    }
+  }
+
+  if (persisted.type === "tcvdb" && current.type === "tcvdb") {
+    if (persisted.tcvdb?.url !== current.tcvdb?.url) {
+      diffs.push(`tcvdb url changed: ${persisted.tcvdb?.url} → ${current.tcvdb?.url}`);
+    }
+    if (persisted.tcvdb?.database !== current.tcvdb?.database) {
+      diffs.push(`tcvdb database changed: ${persisted.tcvdb?.database} → ${current.tcvdb?.database}`);
+    }
+  }
+
+  return diffs;
+}

src/utils/memory-cleaner.ts

@@ -1,7 +1,7 @@
 import fs from "node:fs/promises";
 import path from "node:path";
 
-import type { VectorStore } from "../store/vector-store.js";
+import type { IMemoryStore } from "../store/types.js";
 import { ManagedTimer } from "./managed-timer.js";
 
 interface Logger {
@@ -16,7 +16,7 @@ export interface MemoryCleanerOptions {
   retentionDays: number;
   cleanTime: string;
   logger?: Logger;
-  vectorStore?: VectorStore;
+  vectorStore?: IMemoryStore;
 }
 
 interface CleanupStats {
@@ -33,14 +33,14 @@ const L1_DIR_NAME = "records";
 export class LocalMemoryCleaner {
   private readonly timer: ManagedTimer;
   private destroyed = false;
-  private vectorStore?: VectorStore;
+  private vectorStore?: IMemoryStore;
 
   constructor(private readonly opts: MemoryCleanerOptions) {
     this.timer = new ManagedTimer("memory-tdai-cleaner", () => this.destroyed);
     this.vectorStore = opts.vectorStore;
   }
 
-  setVectorStore(vectorStore: VectorStore | undefined): void {
+  setVectorStore(vectorStore: IMemoryStore | undefined): void {
     this.vectorStore = vectorStore;
   }
 
@@ -51,10 +51,10 @@ export class LocalMemoryCleaner {
     const tz = Intl.DateTimeFormat().resolvedOptions().timeZone || "unknown";
     const utcOffset = formatUtcOffset(-now.getTimezoneOffset());
 
-    this.opts.logger?.info(
+    this.opts.logger?.debug?.(
       `${TAG} Enabled: retentionDays=${this.opts.retentionDays}, cleanTime=${this.opts.cleanTime}, dirs=[${L0_DIR_NAME}, ${L1_DIR_NAME}]`,
     );
-    this.opts.logger?.info(
+    this.opts.logger?.debug?.(
       `${TAG} Runtime clock: nowLocal=${formatLocalDateTime(now)}, nowIso=${now.toISOString()}, tz=${tz}, utcOffset=${utcOffset}`,
     );
 
@@ -110,7 +110,7 @@ export class LocalMemoryCleaner {
       let failedL1DbCleanup = 0;
 
       try {
-        removedL0 = vectorStore.deleteL0ExpiredByRecordedAt(cutoffIso);
+        removedL0 = await vectorStore.deleteL0Expired(cutoffIso);
       } catch (err) {
         failedL0DbCleanup = 1;
         this.opts.logger?.warn(
@@ -119,7 +119,7 @@ export class LocalMemoryCleaner {
       }
 
       try {
-        removedL1 = vectorStore.deleteL1ExpiredByUpdatedTime(cutoffIso);
+        removedL1 = await vectorStore.deleteL1Expired(cutoffIso);
       } catch (err) {
         failedL1DbCleanup = 1;
         this.opts.logger?.warn(
@@ -150,7 +150,7 @@ export class LocalMemoryCleaner {
     const passedToday = targetToday <= nowMs;
     const delayMs = Math.max(0, next - nowMs);
 
-    this.opts.logger?.info(
+    this.opts.logger?.debug?.(
       `${TAG} Schedule next run: nowLocal=${formatLocalDateTime(now)}, cleanTime=${this.opts.cleanTime}, targetTodayLocal=${formatLocalDateTime(new Date(targetToday))}, passedToday=${passedToday}, nextRunLocal=${formatLocalDateTime(new Date(next))}, nextRunIso=${new Date(next).toISOString()}, delayMs=${delayMs}`,
     );
 

src/utils/pipeline-factory.ts

@@ -0,0 +1,720 @@
+/**
+ * Pipeline factory: shared infrastructure for creating and wiring
+ * MemoryPipelineManager instances with VectorStore, EmbeddingService,
+ * L1 runner, L2 runner, L3 runner, and persister.
+ *
+ * Used by both:
+ * - `index.ts` (live plugin runtime)
+ * - `seed-runtime.ts` (standalone seed CLI command)
+ *
+ * This avoids duplicating VectorStore init, L1/L2/L3 extraction logic,
+ * persister wiring, and destroy sequences across multiple callers.
+ */
+
+import fs from "node:fs";
+import path from "node:path";
+import type { MemoryTdaiConfig } from "../config.js";
+import { MemoryPipelineManager } from "./pipeline-manager.js";
+import type { L2Runner, L3Runner } from "./pipeline-manager.js";
+import { SessionFilter } from "./session-filter.js";
+import { extractL1Memories } from "../record/l1-extractor.js";
+import { readConversationMessagesGroupedBySessionId } from "../conversation/l0-recorder.js";
+import type { ConversationMessage } from "../conversation/l0-recorder.js";
+import { CheckpointManager } from "./checkpoint.js";
+import type { PipelineSessionState } from "./checkpoint.js";
+import { createStoreBundle } from "../store/factory.js";
+import type { IMemoryStore } from "../store/types.js";
+import type { EmbeddingService } from "../store/embedding.js";
+import {
+  readManifest,
+  writeManifest,
+  buildStoreInfo,
+  diffStoreBinding,
+  type Manifest,
+} from "./manifest.js";
+import { SceneExtractor } from "../scene/scene-extractor.js";
+import { PersonaTrigger } from "../persona/persona-trigger.js";
+import { PersonaGenerator } from "../persona/persona-generator.js";
+import { pullProfilesToLocal, syncLocalProfilesToStore } from "../profile/profile-sync.js";
+
+const TAG = "[memory-tdai] [pipeline-factory]";
+
+function supportsProfileSyncWrite(store?: IMemoryStore): boolean {
+  return !!(store?.syncProfiles || store?.deleteProfiles);
+}
+
+// ============================
+// Logger interface
+// ============================
+
+export interface PipelineLogger {
+  debug?: (message: string) => void;
+  info: (message: string) => void;
+  warn: (message: string) => void;
+  error: (message: string) => void;
+}
+
+// ============================
+// Factory options
+// ============================
+
+export interface PipelineFactoryOptions {
+  /** Plugin data directory (L0, records, scene_blocks, vectors.db, etc.). */
+  pluginDataDir: string;
+  /** Parsed memory-tdai config. */
+  cfg: MemoryTdaiConfig;
+  /** OpenClaw config object (needed for LLM calls in L1). */
+  openclawConfig: unknown;
+  /** Logger instance. */
+  logger: PipelineLogger;
+  /** Session filter (optional, defaults to empty). */
+  sessionFilter?: SessionFilter;
+}
+
+// ============================
+// Factory result
+// ============================
+
+export interface PipelineInstance {
+  /** The pipeline scheduler. */
+  scheduler: MemoryPipelineManager;
+  /** VectorStore (undefined if init failed or degraded). */
+  vectorStore: IMemoryStore | undefined;
+  /** EmbeddingService (undefined if not configured or init failed). */
+  embeddingService: EmbeddingService | undefined;
+  /**
+   * Destroy all resources (scheduler, VectorStore, EmbeddingService).
+   * Call this on shutdown / cleanup.
+   */
+  destroy: () => Promise<void>;
+}
+
+// ============================
+// Data directory init
+// ============================
+
+/**
+ * Ensure all required data subdirectories exist under `pluginDataDir`.
+ * Safe to call multiple times (mkdirSync with `recursive: true`).
+ */
+export function initDataDirectories(dataDir: string): void {
+  const dirs = ["conversations", "records", "scene_blocks", ".metadata", ".backup"];
+  for (const sub of dirs) {
+    fs.mkdirSync(path.join(dataDir, sub), { recursive: true });
+  }
+}
+
+// ============================
+// Store init (once-async singleton)
+// ============================
+
+export interface StoreInitResult {
+  vectorStore: IMemoryStore | undefined;
+  embeddingService: EmbeddingService | undefined;
+  /** Whether a background re-index is needed (embedding config changed). */
+  needsReindex: boolean;
+  reindexReason?: string;
+}
+
+/**
+ * Cached store init promises — keyed by `pluginDataDir` so that different
+ * data directories (e.g. live runtime vs. seed output) each get their own
+ * store instance, while concurrent callers for the *same* directory share
+ * one initialization.
+ */
+const _storeInitCache = new Map<string, Promise<StoreInitResult>>();
+
+/**
+ * Initialize store backend and (optionally) EmbeddingService.
+ *
+ * **Once-async semantics per dataDir**: the first call for a given
+ * `pluginDataDir` creates the store and caches the result; subsequent
+ * calls with the same dir return the cached Promise immediately.
+ * Call `resetStores()` during shutdown to clear the cache.
+ *
+ * Supports both SQLite (sync init) and TCVDB (async init) backends.
+ */
+export function initStores(
+  cfg: MemoryTdaiConfig,
+  pluginDataDir: string,
+  logger: PipelineLogger,
+): Promise<StoreInitResult> {
+  const key = pluginDataDir;
+  if (!_storeInitCache.has(key)) {
+    _storeInitCache.set(key, _doInitStores(cfg, pluginDataDir, logger));
+  }
+  return _storeInitCache.get(key)!;
+}
+
+/**
+ * Reset the cached store singleton(s).
+ *
+ * Call this during `gateway_stop` (after closing the actual store/embedding
+ * resources) so that a subsequent `register()` on hot-restart can
+ * re-initialize fresh instances.
+ *
+ * @param pluginDataDir  If provided, only clear the cache for that dir.
+ *                       If omitted, clear all cached stores.
+ */
+export function resetStores(pluginDataDir?: string): void {
+  if (pluginDataDir) {
+    _storeInitCache.delete(pluginDataDir);
+  } else {
+    _storeInitCache.clear();
+  }
+}
+
+/**
+ * Internal: actual store initialization logic (called once by the cache).
+ */
+async function _doInitStores(
+  cfg: MemoryTdaiConfig,
+  pluginDataDir: string,
+  logger: PipelineLogger,
+): Promise<StoreInitResult> {
+  let vectorStore: IMemoryStore | undefined;
+  let embeddingService: EmbeddingService | undefined;
+  let needsReindex = false;
+  let reindexReason: string | undefined;
+
+  try {
+    const bundle = createStoreBundle(cfg, {
+      dataDir: pluginDataDir,
+      logger,
+    });
+    vectorStore = bundle.store;
+    embeddingService = bundle.embedding ?? undefined;
+
+    const providerInfo = embeddingService?.getProviderInfo();
+    const initResult = await vectorStore.init(providerInfo);
+
+    if (vectorStore.isDegraded()) {
+      logger.warn(`${TAG} Store is in degraded mode, falling back to keyword dedup`);
+      vectorStore = undefined;
+      embeddingService = undefined;
+    } else {
+      logger.debug?.(
+        `${TAG} Store initialized: backend=${cfg.storeBackend}, provider=${cfg.embedding.provider}`,
+      );
+      needsReindex = initResult.needsReindex;
+      reindexReason = initResult.reason;
+
+      // ── Manifest: first-write + config-drift detection ──
+      try {
+        const currentStoreInfo = buildStoreInfo(bundle.storeSnapshot);
+        const existing = readManifest(pluginDataDir);
+
+        if (!existing) {
+          // First init — write manifest
+          const manifest: Manifest = {
+            version: 1,
+            createdAt: new Date().toISOString(),
+            store: currentStoreInfo,
+            seed: null,
+          };
+          writeManifest(pluginDataDir, manifest);
+          logger.debug?.(`${TAG} Manifest created: ${JSON.stringify(currentStoreInfo)}`);
+        } else {
+          // Compare persisted store binding against current config
+          const diffs = diffStoreBinding(existing.store, currentStoreInfo);
+          if (diffs.length > 0) {
+            logger.warn(
+              `${TAG} ⚠️ Store config has changed since this data directory was created! ` +
+              `Diffs: ${diffs.join("; ")}. ` +
+              `Local JSONL data may not match the current store. ` +
+              `Consider re-seeding or migrating data.`,
+            );
+          }
+        }
+      } catch (err) {
+        logger.warn(`${TAG} Failed to read/write manifest (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
+      }
+    }
+  } catch (err) {
+    logger.warn(
+      `${TAG} Store init failed; vector/FTS recall and dedup conflict detection will be unavailable: ${err instanceof Error ? err.message : String(err)}`,
+    );
+    vectorStore = undefined;
+    embeddingService = undefined;
+  }
+
+  return { vectorStore, embeddingService, needsReindex, reindexReason };
+}
+
+// ============================
+// L1 Runner factory
+// ============================
+
+/**
+ * Create the standard L1 runner function.
+ *
+ * Reads L0 messages (from VectorStore DB or JSONL fallback), groups by sessionId,
+ * runs extractL1Memories for each group, and updates the checkpoint cursor.
+ */
+export function createL1Runner(opts: {
+  pluginDataDir: string;
+  cfg: MemoryTdaiConfig;
+  openclawConfig: unknown;
+  vectorStore: IMemoryStore | undefined;
+  embeddingService: EmbeddingService | undefined;
+  logger: PipelineLogger;
+  /**
+   * Getter for the plugin instance ID used for metric reporting.
+   * Called at runner execution time (not at creation time) so that the ID is
+   * available even when the runner is wired before instanceId is resolved.
+   * Metrics are skipped when the getter returns undefined.
+   */
+  getInstanceId?: () => string | undefined;
+}): (params: { sessionKey: string }) => Promise<{ processedCount: number }> {
+  const { pluginDataDir, cfg, openclawConfig, vectorStore, embeddingService, logger, getInstanceId } = opts;
+  const config = openclawConfig as Record<string, unknown> | undefined;
+
+  return async ({ sessionKey }) => {
+    if (!config) {
+      logger.debug?.(`${TAG} [l1] No OpenClaw config, skipping L1 extraction`);
+      return { processedCount: 0 };
+    }
+
+    const checkpoint = new CheckpointManager(pluginDataDir, logger);
+    const cp = await checkpoint.read();
+    const runnerState = checkpoint.getRunnerState(cp, sessionKey);
+
+    logger.info(
+      `${TAG} [l1] Session ${sessionKey}: l1_cursor=${runnerState.last_l1_cursor || "(start)"}`,
+    );
+
+    try {
+      let groups: Array<{ sessionId: string; messages: ConversationMessage[] }>;
+      let maxRecordedAtMs = 0;
+
+      if (vectorStore && !vectorStore.isDegraded()) {
+        const l1Cursor = runnerState.last_l1_cursor > 0
+          ? runnerState.last_l1_cursor
+          : undefined;
+        const dbGroups = await vectorStore.queryL0GroupedBySessionId(sessionKey, l1Cursor);
+        groups = dbGroups.map((g) => ({
+          sessionId: g.sessionId,
+          messages: g.messages.map((m) => ({
+            id: m.id,
+            role: m.role as "user" | "assistant",
+            content: m.content,
+            timestamp: m.timestamp,
+          })),
+        }));
+        // Compute max recordedAtMs across all groups for cursor advancement
+        for (const g of dbGroups) {
+          for (const m of g.messages) {
+            if (m.recordedAtMs > maxRecordedAtMs) maxRecordedAtMs = m.recordedAtMs;
+          }
+        }
+        logger.debug?.(`${TAG} [l1] L0 data source: VectorStore DB`);
+      } else {
+        logger.debug?.(`${TAG} [l1] L0 data source: JSONL files (VectorStore unavailable)`);
+        const jsonlGroups = await readConversationMessagesGroupedBySessionId(
+          sessionKey,
+          pluginDataDir,
+          runnerState.last_l1_cursor || undefined,
+          logger,
+          50,
+        );
+        groups = jsonlGroups.map((g) => ({
+          sessionId: g.sessionId,
+          messages: g.messages,
+        }));
+        // Compute max recordedAtMs from JSONL groups
+        for (const g of jsonlGroups) {
+          for (const m of g.messages) {
+            if (m.recordedAtMs > maxRecordedAtMs) maxRecordedAtMs = m.recordedAtMs;
+          }
+        }
+      }
+
+      if (groups.length === 0) {
+        logger.debug?.(`${TAG} [l1] No new L0 messages for session ${sessionKey}`);
+        return { processedCount: 0 };
+      }
+
+      const totalMessages = groups.reduce((sum, g) => sum + g.messages.length, 0);
+      logger.info(
+        `${TAG} [l1] Processing ${totalMessages} L0 messages across ${groups.length} sessionId group(s) for session ${sessionKey}`,
+      );
+
+      let totalExtracted = 0;
+      let totalStored = 0;
+      let lastSceneName: string | undefined;
+
+      for (const group of groups) {
+        logger.debug?.(
+          `${TAG} [l1] Group sessionId=${group.sessionId || "(empty)"}: ${group.messages.length} messages`,
+        );
+
+        const l1Result = await extractL1Memories({
+          messages: group.messages,
+          sessionKey,
+          sessionId: group.sessionId,
+          baseDir: pluginDataDir,
+          config,
+          options: {
+            enableDedup: cfg.extraction.enableDedup,
+            maxMemoriesPerSession: cfg.extraction.maxMemoriesPerSession,
+            model: cfg.extraction.model,
+            previousSceneName: lastSceneName ?? (runnerState.last_scene_name || undefined),
+            vectorStore,
+            embeddingService,
+            conflictRecallTopK: cfg.embedding.conflictRecallTopK,
+          },
+          logger,
+          instanceId: getInstanceId?.(),
+        });
+
+        totalExtracted += l1Result.extractedCount;
+        totalStored += l1Result.storedCount;
+        if (l1Result.lastSceneName) {
+          lastSceneName = l1Result.lastSceneName;
+        }
+      }
+
+      // Use maxRecordedAtMs (write time) as cursor — always positive, TCVDB-safe
+      await checkpoint.markL1ExtractionComplete(sessionKey, totalStored, maxRecordedAtMs || undefined, lastSceneName);
+      logger.info(
+        `${TAG} [l1] L1 complete: extracted=${totalExtracted}, stored=${totalStored} (${groups.length} group(s))`,
+      );
+
+      return { processedCount: totalMessages };
+    } catch (err) {
+      logger.error(`${TAG} [l1] L1 failed: ${err instanceof Error ? err.stack ?? err.message : String(err)}`);
+      throw err;
+    }
+  };
+}
+
+// ============================
+// Persister factory
+// ============================
+
+/**
+ * Create the standard pipeline state persister.
+ * Saves pipeline session states to the checkpoint file.
+ */
+export function createPersister(
+  pluginDataDir: string,
+  logger: PipelineLogger,
+): (states: Record<string, PipelineSessionState>) => Promise<void> {
+  return async (states) => {
+    const checkpoint = new CheckpointManager(pluginDataDir, logger);
+    await checkpoint.mergePipelineStates(states);
+  };
+}
+
+// ============================
+// L2 Runner factory
+// ============================
+
+/**
+ * Create the standard L2 runner function (scene extraction).
+ *
+ * Reads L1 memory records (incremental via VectorStore or JSONL fallback),
+ * runs SceneExtractor, and returns the latest cursor for pipeline-manager
+ * to track incremental progress.
+ *
+ * Used by both `index.ts` (live runtime) and `seed-runtime.ts` (seed CLI).
+ */
+export function createL2Runner(opts: {
+  pluginDataDir: string;
+  cfg: MemoryTdaiConfig;
+  openclawConfig: unknown;
+  vectorStore: IMemoryStore | undefined;
+  logger: PipelineLogger;
+  instanceId?: string;
+}): L2Runner {
+  const { pluginDataDir, cfg, openclawConfig, vectorStore, logger, instanceId } = opts;
+  let profileBaseline = new Map<string, { version: number; contentMd5: string; createdAtMs: number }>();
+
+  return async (sessionKey: string, cursor?: string) => {
+    logger.debug?.(
+      `${TAG} [L2] session=${sessionKey}, updatedAfter=${cursor ?? "(full)"}`,
+    );
+
+    let records: Array<{ content: string; created_at: string; id: string; updatedAt: string }>;
+
+    if (vectorStore?.pullProfiles && !vectorStore.isDegraded()) {
+      profileBaseline = await pullProfilesToLocal(pluginDataDir, vectorStore, logger);
+    }
+
+    if (vectorStore && !vectorStore.isDegraded()) {
+      const { queryMemoryRecords } = await import("../record/l1-reader.js");
+      const memRecords = await queryMemoryRecords(vectorStore, {
+        sessionKey,
+        updatedAfter: cursor,
+      }, logger);
+
+      if (memRecords.length === 0) {
+        logger.debug?.(
+          `${TAG} [L2] No new L1 records since cursor (session=${sessionKey}, updatedAfter=${cursor ?? "(full)"}), skipping scene extraction`,
+        );
+        return;
+      }
+
+      logger.debug?.(
+        `${TAG} [L2] Incremental query returned ${memRecords.length} record(s) (session=${sessionKey})`,
+      );
+
+      records = memRecords.map((r) => ({
+        content: r.content,
+        created_at: r.createdAt,
+        id: r.id,
+        updatedAt: r.updatedAt,
+      }));
+    } else {
+      logger.debug?.(`${TAG} [L2] VectorStore unavailable, falling back to JSONL read (session=${sessionKey})`);
+      const { readMemoryRecords } = await import("../record/l1-reader.js");
+      let sessionRecords = await readMemoryRecords(sessionKey, pluginDataDir, logger);
+
+      if (cursor) {
+        const beforeCount = sessionRecords.length;
+        sessionRecords = sessionRecords.filter((r) => {
+          const t = r.updatedAt || r.createdAt || "";
+          return t > cursor;
+        });
+        logger.debug?.(
+          `${TAG} [L2] JSONL time filter: ${beforeCount} → ${sessionRecords.length} record(s) (updatedAfter=${cursor})`,
+        );
+      }
+
+      if (sessionRecords.length === 0) {
+        logger.debug?.(`${TAG} [L2] No new L1 records found (JSONL fallback, session=${sessionKey}), skipping scene extraction`);
+        return;
+      }
+
+      records = sessionRecords.map((r) => ({
+        content: r.content,
+        created_at: r.createdAt,
+        id: r.id,
+        updatedAt: r.updatedAt,
+      }));
+    }
+
+    const extractor = new SceneExtractor({
+      dataDir: pluginDataDir,
+      config: openclawConfig!,
+      model: cfg.persona.model,
+      maxScenes: cfg.persona.maxScenes,
+      sceneBackupCount: cfg.persona.sceneBackupCount,
+      logger,
+      instanceId,
+    });
+
+    const memories = records.map((r) => ({
+      content: r.content,
+      created_at: r.created_at,
+      id: r.id,
+    }));
+
+    const preCheckpoint = new CheckpointManager(pluginDataDir, logger);
+    const preState = await preCheckpoint.read();
+    const preScenesProcessed = preState.scenes_processed;
+    const preMemoriesSince = preState.memories_since_last_persona;
+    const preTotalProcessed = preState.total_processed;
+
+    const extractResult = await extractor.extract(memories);
+    if (extractResult.success && extractResult.memoriesProcessed > 0) {
+      const checkpoint = new CheckpointManager(pluginDataDir, logger);
+      const postState = await checkpoint.read();
+      if (
+        postState.scenes_processed < preScenesProcessed ||
+        postState.total_processed < preTotalProcessed
+      ) {
+        logger.warn(
+          `${TAG} [L2] ⚠️ Checkpoint corruption detected! ` +
+          `scenes_processed: ${preScenesProcessed} → ${postState.scenes_processed}, ` +
+          `total_processed: ${preTotalProcessed} → ${postState.total_processed}, ` +
+          `memories_since: ${preMemoriesSince} → ${postState.memories_since_last_persona}. ` +
+          `Repairing...`,
+        );
+        await checkpoint.write({
+          ...postState,
+          scenes_processed: Math.max(postState.scenes_processed, preScenesProcessed),
+          total_processed: Math.max(postState.total_processed, preTotalProcessed),
+          memories_since_last_persona: Math.max(postState.memories_since_last_persona, preMemoriesSince),
+        });
+        logger.info(`${TAG} [L2] Checkpoint repaired`);
+      }
+
+      if (vectorStore && supportsProfileSyncWrite(vectorStore)) {
+        await syncLocalProfilesToStore(pluginDataDir, vectorStore, profileBaseline, logger);
+      }
+      await checkpoint.incrementScenesProcessed();
+
+      const latestCursor = records.reduce((latest, r) => {
+        return r.updatedAt > latest ? r.updatedAt : latest;
+      }, "");
+
+      logger.debug?.(
+        `${TAG} [L2] Extraction complete: processed=${extractResult.memoriesProcessed}, latestCursor=${latestCursor}`,
+      );
+
+      return { latestCursor: latestCursor || undefined };
+    }
+  };
+}
+
+// ============================
+// L3 Runner factory
+// ============================
+
+/**
+ * Create the standard L3 runner function (persona generation).
+ *
+ * Uses PersonaTrigger to check if generation is needed, then runs
+ * PersonaGenerator. Used by both `index.ts` and `seed-runtime.ts`.
+ */
+export function createL3Runner(opts: {
+  pluginDataDir: string;
+  cfg: MemoryTdaiConfig;
+  openclawConfig: unknown;
+  vectorStore?: IMemoryStore;
+  logger: PipelineLogger;
+  instanceId?: string;
+}): L3Runner {
+  const { pluginDataDir, cfg, openclawConfig, vectorStore, logger, instanceId } = opts;
+
+  return async () => {
+    const trigger = new PersonaTrigger({
+      dataDir: pluginDataDir,
+      interval: cfg.persona.triggerEveryN,
+      logger,
+    });
+
+    const { should, reason } = await trigger.shouldGenerate();
+    if (!should) {
+      logger.debug?.(`${TAG} [L3] Persona generation not needed`);
+      return;
+    }
+
+    if (!openclawConfig) {
+      logger.warn(`${TAG} [L3] No OpenClaw config, skipping persona generation`);
+      return;
+    }
+
+    // Pull remote profiles to establish fresh baseline before generation.
+    // This ensures syncLocalProfilesToStore() has correct baselineVersion
+    // for the optimistic-lock check instead of defaulting to 0.
+    let profileBaseline = new Map<string, { version: number; contentMd5: string; createdAtMs: number }>();
+    if (vectorStore?.pullProfiles && !vectorStore.isDegraded()) {
+      profileBaseline = await pullProfilesToLocal(pluginDataDir, vectorStore, logger);
+    }
+
+    logger.info(`${TAG} [L3] Starting persona generation: ${reason}`);
+    const generator = new PersonaGenerator({
+      dataDir: pluginDataDir,
+      config: openclawConfig,
+      model: cfg.persona.model,
+      backupCount: cfg.persona.backupCount,
+      logger,
+      instanceId,
+    });
+    const genResult = await generator.generateLocalPersona(reason);
+    if (!genResult) {
+      logger.info(`${TAG} [L3] Persona generation skipped (no changes)`);
+      return;
+    }
+
+    if (vectorStore && supportsProfileSyncWrite(vectorStore)) {
+      await syncLocalProfilesToStore(pluginDataDir, vectorStore, profileBaseline, logger);
+    }
+
+    const checkpoint = new CheckpointManager(pluginDataDir, logger);
+    const cp = await checkpoint.read();
+    await checkpoint.markPersonaGenerated(cp.total_processed);
+    logger.info(`${TAG} [L3] Persona generation succeeded`);
+  };
+}
+
+// ============================
+// Pipeline Manager factory
+// ============================
+
+/**
+ * Create a MemoryPipelineManager with the standard config mapping.
+ */
+export function createPipelineManager(
+  cfg: MemoryTdaiConfig,
+  logger: PipelineLogger,
+  sessionFilter?: SessionFilter,
+): MemoryPipelineManager {
+  return new MemoryPipelineManager(
+    {
+      everyNConversations: cfg.pipeline.everyNConversations,
+      enableWarmup: cfg.pipeline.enableWarmup,
+      l1: { idleTimeoutSeconds: cfg.pipeline.l1IdleTimeoutSeconds },
+      l2: {
+        delayAfterL1Seconds: cfg.pipeline.l2DelayAfterL1Seconds,
+        minIntervalSeconds: cfg.pipeline.l2MinIntervalSeconds,
+        maxIntervalSeconds: cfg.pipeline.l2MaxIntervalSeconds,
+        sessionActiveWindowHours: cfg.pipeline.sessionActiveWindowHours,
+      },
+    },
+    logger,
+    sessionFilter ?? new SessionFilter([]),
+  );
+}
+
+// ============================
+// Full pipeline factory
+// ============================
+
+/**
+ * Create a fully wired pipeline instance: VectorStore + EmbeddingService +
+ * MemoryPipelineManager with L1 runner and persister attached.
+ *
+ * This is the high-level entry point used by both `index.ts` and `seed-runtime.ts`.
+ * Callers should attach L2/L3 runners after creation using `createL2Runner()`
+ * and `createL3Runner()` from this module.
+ */
+export async function createPipeline(opts: PipelineFactoryOptions): Promise<PipelineInstance> {
+  const { pluginDataDir, cfg, openclawConfig, logger, sessionFilter } = opts;
+
+  // Ensure data directories exist
+  initDataDirectories(pluginDataDir);
+
+  // Initialize stores (once-async: reuses cached result if already initialized)
+  const stores = await initStores(cfg, pluginDataDir, logger);
+  const { vectorStore, embeddingService } = stores;
+
+  // Create pipeline manager
+  const scheduler = createPipelineManager(cfg, logger, sessionFilter);
+
+  // Wire L1 runner
+  scheduler.setL1Runner(createL1Runner({
+    pluginDataDir,
+    cfg,
+    openclawConfig,
+    vectorStore,
+    embeddingService,
+    logger,
+  }));
+
+  // Wire persister
+  scheduler.setPersister(createPersister(pluginDataDir, logger));
+
+  // Destroy function
+  const destroy = async () => {
+    logger.info(`${TAG} Destroying pipeline...`);
+    await scheduler.destroy();
+    if (vectorStore) {
+      logger.info(`${TAG} Closing VectorStore`);
+      vectorStore.close();
+    }
+    if (embeddingService?.close) {
+      try {
+        logger.info(`${TAG} Closing EmbeddingService`);
+        await embeddingService.close();
+      } catch (err) {
+        logger.warn(`${TAG} Error closing EmbeddingService: ${err instanceof Error ? err.message : String(err)}`);
+      }
+    }
+    logger.info(`${TAG} Pipeline destroyed`);
+  };
+
+  return { scheduler, vectorStore, embeddingService, destroy };
+}

src/utils/pipeline-manager.ts

@@ -262,7 +262,7 @@ export class MemoryPipelineManager {
     this.logger = logger;
     this.sessionFilter = sessionFilter ?? new SessionFilter();
 
-    this.logger?.info(
+    this.logger?.debug?.(
       `${TAG} Initialized: everyNConversations=${config.everyNConversations}, ` +
       `warmup=${config.enableWarmup ? "enabled" : "disabled"}, ` +
       `l1IdleTimeout=${config.l1.idleTimeoutSeconds}s, ` +
@@ -1076,12 +1076,22 @@ export class MemoryPipelineManager {
     return this.destroyed;
   }
 
-  /** Queue sizes for monitoring. */
-  getQueueSizes(): { l1: number; l2: number; l3: number } {
+  /** Queue sizes and running state for monitoring. */
+  getQueueSizes(): {
+    l1: number; l2: number; l3: number;
+    l1Pending: boolean; l2Pending: boolean; l3Pending: boolean;
+    l1Idle: boolean; l2Idle: boolean; l3Idle: boolean;
+  } {
     return {
       l1: this.l1Queue.size,
       l2: this.l2Queue.size,
       l3: this.l3Queue.size,
+      l1Pending: this.l1Queue.pending,
+      l2Pending: this.l2Queue.pending,
+      l3Pending: this.l3Queue.pending,
+      l1Idle: this.l1Queue.idle,
+      l2Idle: this.l2Queue.idle,
+      l3Idle: this.l3Queue.idle,
     };
   }
 }

src/utils/sanitize.ts

@@ -38,6 +38,9 @@ export function sanitizeText(text: string): string {
   // Remove framework reply directive tags: [[reply_to_current]], [[reply_to_xxx]], etc.
   cleaned = cleaned.replace(/\[\[reply_to[^\]]*\]\]\s*/g, "");
 
+  // Remove injected skill-selection wrappers, e.g. ¥¥[... ]¥¥
+  cleaned = cleaned.replace(/¥¥\[[\s\S]*?\]¥¥/g, "");
+
   // Remove line-leading timestamps, e.g. "[Tue 2026-03-24 03:48 UTC]"
   // or "[Tue 2026-03-24 20:21 GMT+8]", "[Thu 2026-03-24 01:51 GMT+5:30]"
   // Matches brackets containing word chars, digits, hyphens, colons, plus signs,

src/utils/serial-queue.ts

@@ -50,6 +50,11 @@ export class SerialQueue {
     return this.running;
   }
 
+  /** Whether the queue is idle (no queued tasks and nothing running). */
+  get idle(): boolean {
+    return this.queue.length === 0 && !this.running;
+  }
+
   /** Add a task to the queue. Returns the task's result promise. */
   add<T>(task: Task<T>): Promise<T> {
     return new Promise<T>((resolve, reject) => {