feat: release v0.2.2 — TCVDB backend, BM25 hybrid retrieval, pipeline refactor

2026-07-28 13:14:32 +00:00 · 2026-05-13 01:23:05 +08:00
parent 5bf5f890a3
commit a74b0b3e43
45 changed files with 8247 additions and 756 deletions
@@ -0,0 +1,720 @@
+/**
+ * Pipeline factory: shared infrastructure for creating and wiring
+ * MemoryPipelineManager instances with VectorStore, EmbeddingService,
+ * L1 runner, L2 runner, L3 runner, and persister.
+ *
+ * Used by both:
+ * - `index.ts` (live plugin runtime)
+ * - `seed-runtime.ts` (standalone seed CLI command)
+ *
+ * This avoids duplicating VectorStore init, L1/L2/L3 extraction logic,
+ * persister wiring, and destroy sequences across multiple callers.
+ */
+
+import fs from "node:fs";
+import path from "node:path";
+import type { MemoryTdaiConfig } from "../config.js";
+import { MemoryPipelineManager } from "./pipeline-manager.js";
+import type { L2Runner, L3Runner } from "./pipeline-manager.js";
+import { SessionFilter } from "./session-filter.js";
+import { extractL1Memories } from "../record/l1-extractor.js";
+import { readConversationMessagesGroupedBySessionId } from "../conversation/l0-recorder.js";
+import type { ConversationMessage } from "../conversation/l0-recorder.js";
+import { CheckpointManager } from "./checkpoint.js";
+import type { PipelineSessionState } from "./checkpoint.js";
+import { createStoreBundle } from "../store/factory.js";
+import type { IMemoryStore } from "../store/types.js";
+import type { EmbeddingService } from "../store/embedding.js";
+import {
+  readManifest,
+  writeManifest,
+  buildStoreInfo,
+  diffStoreBinding,
+  type Manifest,
+} from "./manifest.js";
+import { SceneExtractor } from "../scene/scene-extractor.js";
+import { PersonaTrigger } from "../persona/persona-trigger.js";
+import { PersonaGenerator } from "../persona/persona-generator.js";
+import { pullProfilesToLocal, syncLocalProfilesToStore } from "../profile/profile-sync.js";
+
+const TAG = "[memory-tdai] [pipeline-factory]";
+
+function supportsProfileSyncWrite(store?: IMemoryStore): boolean {
+  return !!(store?.syncProfiles || store?.deleteProfiles);
+}
+
+// ============================
+// Logger interface
+// ============================
+
+export interface PipelineLogger {
+  debug?: (message: string) => void;
+  info: (message: string) => void;
+  warn: (message: string) => void;
+  error: (message: string) => void;
+}
+
+// ============================
+// Factory options
+// ============================
+
+export interface PipelineFactoryOptions {
+  /** Plugin data directory (L0, records, scene_blocks, vectors.db, etc.). */
+  pluginDataDir: string;
+  /** Parsed memory-tdai config. */
+  cfg: MemoryTdaiConfig;
+  /** OpenClaw config object (needed for LLM calls in L1). */
+  openclawConfig: unknown;
+  /** Logger instance. */
+  logger: PipelineLogger;
+  /** Session filter (optional, defaults to empty). */
+  sessionFilter?: SessionFilter;
+}
+
+// ============================
+// Factory result
+// ============================
+
+export interface PipelineInstance {
+  /** The pipeline scheduler. */
+  scheduler: MemoryPipelineManager;
+  /** VectorStore (undefined if init failed or degraded). */
+  vectorStore: IMemoryStore | undefined;
+  /** EmbeddingService (undefined if not configured or init failed). */
+  embeddingService: EmbeddingService | undefined;
+  /**
+   * Destroy all resources (scheduler, VectorStore, EmbeddingService).
+   * Call this on shutdown / cleanup.
+   */
+  destroy: () => Promise<void>;
+}
+
+// ============================
+// Data directory init
+// ============================
+
+/**
+ * Ensure all required data subdirectories exist under `pluginDataDir`.
+ * Safe to call multiple times (mkdirSync with `recursive: true`).
+ */
+export function initDataDirectories(dataDir: string): void {
+  const dirs = ["conversations", "records", "scene_blocks", ".metadata", ".backup"];
+  for (const sub of dirs) {
+    fs.mkdirSync(path.join(dataDir, sub), { recursive: true });
+  }
+}
+
+// ============================
+// Store init (once-async singleton)
+// ============================
+
+export interface StoreInitResult {
+  vectorStore: IMemoryStore | undefined;
+  embeddingService: EmbeddingService | undefined;
+  /** Whether a background re-index is needed (embedding config changed). */
+  needsReindex: boolean;
+  reindexReason?: string;
+}
+
+/**
+ * Cached store init promises — keyed by `pluginDataDir` so that different
+ * data directories (e.g. live runtime vs. seed output) each get their own
+ * store instance, while concurrent callers for the *same* directory share
+ * one initialization.
+ */
+const _storeInitCache = new Map<string, Promise<StoreInitResult>>();
+
+/**
+ * Initialize store backend and (optionally) EmbeddingService.
+ *
+ * **Once-async semantics per dataDir**: the first call for a given
+ * `pluginDataDir` creates the store and caches the result; subsequent
+ * calls with the same dir return the cached Promise immediately.
+ * Call `resetStores()` during shutdown to clear the cache.
+ *
+ * Supports both SQLite (sync init) and TCVDB (async init) backends.
+ */
+export function initStores(
+  cfg: MemoryTdaiConfig,
+  pluginDataDir: string,
+  logger: PipelineLogger,
+): Promise<StoreInitResult> {
+  const key = pluginDataDir;
+  if (!_storeInitCache.has(key)) {
+    _storeInitCache.set(key, _doInitStores(cfg, pluginDataDir, logger));
+  }
+  return _storeInitCache.get(key)!;
+}
+
+/**
+ * Reset the cached store singleton(s).
+ *
+ * Call this during `gateway_stop` (after closing the actual store/embedding
+ * resources) so that a subsequent `register()` on hot-restart can
+ * re-initialize fresh instances.
+ *
+ * @param pluginDataDir  If provided, only clear the cache for that dir.
+ *                       If omitted, clear all cached stores.
+ */
+export function resetStores(pluginDataDir?: string): void {
+  if (pluginDataDir) {
+    _storeInitCache.delete(pluginDataDir);
+  } else {
+    _storeInitCache.clear();
+  }
+}
+
+/**
+ * Internal: actual store initialization logic (called once by the cache).
+ */
+async function _doInitStores(
+  cfg: MemoryTdaiConfig,
+  pluginDataDir: string,
+  logger: PipelineLogger,
+): Promise<StoreInitResult> {
+  let vectorStore: IMemoryStore | undefined;
+  let embeddingService: EmbeddingService | undefined;
+  let needsReindex = false;
+  let reindexReason: string | undefined;
+
+  try {
+    const bundle = createStoreBundle(cfg, {
+      dataDir: pluginDataDir,
+      logger,
+    });
+    vectorStore = bundle.store;
+    embeddingService = bundle.embedding ?? undefined;
+
+    const providerInfo = embeddingService?.getProviderInfo();
+    const initResult = await vectorStore.init(providerInfo);
+
+    if (vectorStore.isDegraded()) {
+      logger.warn(`${TAG} Store is in degraded mode, falling back to keyword dedup`);
+      vectorStore = undefined;
+      embeddingService = undefined;
+    } else {
+      logger.debug?.(
+        `${TAG} Store initialized: backend=${cfg.storeBackend}, provider=${cfg.embedding.provider}`,
+      );
+      needsReindex = initResult.needsReindex;
+      reindexReason = initResult.reason;
+
+      // ── Manifest: first-write + config-drift detection ──
+      try {
+        const currentStoreInfo = buildStoreInfo(bundle.storeSnapshot);
+        const existing = readManifest(pluginDataDir);
+
+        if (!existing) {
+          // First init — write manifest
+          const manifest: Manifest = {
+            version: 1,
+            createdAt: new Date().toISOString(),
+            store: currentStoreInfo,
+            seed: null,
+          };
+          writeManifest(pluginDataDir, manifest);
+          logger.debug?.(`${TAG} Manifest created: ${JSON.stringify(currentStoreInfo)}`);
+        } else {
+          // Compare persisted store binding against current config
+          const diffs = diffStoreBinding(existing.store, currentStoreInfo);
+          if (diffs.length > 0) {
+            logger.warn(
+              `${TAG} ⚠️ Store config has changed since this data directory was created! ` +
+              `Diffs: ${diffs.join("; ")}. ` +
+              `Local JSONL data may not match the current store. ` +
+              `Consider re-seeding or migrating data.`,
+            );
+          }
+        }
+      } catch (err) {
+        logger.warn(`${TAG} Failed to read/write manifest (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
+      }
+    }
+  } catch (err) {
+    logger.warn(
+      `${TAG} Store init failed; vector/FTS recall and dedup conflict detection will be unavailable: ${err instanceof Error ? err.message : String(err)}`,
+    );
+    vectorStore = undefined;
+    embeddingService = undefined;
+  }
+
+  return { vectorStore, embeddingService, needsReindex, reindexReason };
+}
+
+// ============================
+// L1 Runner factory
+// ============================
+
+/**
+ * Create the standard L1 runner function.
+ *
+ * Reads L0 messages (from VectorStore DB or JSONL fallback), groups by sessionId,
+ * runs extractL1Memories for each group, and updates the checkpoint cursor.
+ */
+export function createL1Runner(opts: {
+  pluginDataDir: string;
+  cfg: MemoryTdaiConfig;
+  openclawConfig: unknown;
+  vectorStore: IMemoryStore | undefined;
+  embeddingService: EmbeddingService | undefined;
+  logger: PipelineLogger;
+  /**
+   * Getter for the plugin instance ID used for metric reporting.
+   * Called at runner execution time (not at creation time) so that the ID is
+   * available even when the runner is wired before instanceId is resolved.
+   * Metrics are skipped when the getter returns undefined.
+   */
+  getInstanceId?: () => string | undefined;
+}): (params: { sessionKey: string }) => Promise<{ processedCount: number }> {
+  const { pluginDataDir, cfg, openclawConfig, vectorStore, embeddingService, logger, getInstanceId } = opts;
+  const config = openclawConfig as Record<string, unknown> | undefined;
+
+  return async ({ sessionKey }) => {
+    if (!config) {
+      logger.debug?.(`${TAG} [l1] No OpenClaw config, skipping L1 extraction`);
+      return { processedCount: 0 };
+    }
+
+    const checkpoint = new CheckpointManager(pluginDataDir, logger);
+    const cp = await checkpoint.read();
+    const runnerState = checkpoint.getRunnerState(cp, sessionKey);
+
+    logger.info(
+      `${TAG} [l1] Session ${sessionKey}: l1_cursor=${runnerState.last_l1_cursor || "(start)"}`,
+    );
+
+    try {
+      let groups: Array<{ sessionId: string; messages: ConversationMessage[] }>;
+      let maxRecordedAtMs = 0;
+
+      if (vectorStore && !vectorStore.isDegraded()) {
+        const l1Cursor = runnerState.last_l1_cursor > 0
+          ? runnerState.last_l1_cursor
+          : undefined;
+        const dbGroups = await vectorStore.queryL0GroupedBySessionId(sessionKey, l1Cursor);
+        groups = dbGroups.map((g) => ({
+          sessionId: g.sessionId,
+          messages: g.messages.map((m) => ({
+            id: m.id,
+            role: m.role as "user" | "assistant",
+            content: m.content,
+            timestamp: m.timestamp,
+          })),
+        }));
+        // Compute max recordedAtMs across all groups for cursor advancement
+        for (const g of dbGroups) {
+          for (const m of g.messages) {
+            if (m.recordedAtMs > maxRecordedAtMs) maxRecordedAtMs = m.recordedAtMs;
+          }
+        }
+        logger.debug?.(`${TAG} [l1] L0 data source: VectorStore DB`);
+      } else {
+        logger.debug?.(`${TAG} [l1] L0 data source: JSONL files (VectorStore unavailable)`);
+        const jsonlGroups = await readConversationMessagesGroupedBySessionId(
+          sessionKey,
+          pluginDataDir,
+          runnerState.last_l1_cursor || undefined,
+          logger,
+          50,
+        );
+        groups = jsonlGroups.map((g) => ({
+          sessionId: g.sessionId,
+          messages: g.messages,
+        }));
+        // Compute max recordedAtMs from JSONL groups
+        for (const g of jsonlGroups) {
+          for (const m of g.messages) {
+            if (m.recordedAtMs > maxRecordedAtMs) maxRecordedAtMs = m.recordedAtMs;
+          }
+        }
+      }
+
+      if (groups.length === 0) {
+        logger.debug?.(`${TAG} [l1] No new L0 messages for session ${sessionKey}`);
+        return { processedCount: 0 };
+      }
+
+      const totalMessages = groups.reduce((sum, g) => sum + g.messages.length, 0);
+      logger.info(
+        `${TAG} [l1] Processing ${totalMessages} L0 messages across ${groups.length} sessionId group(s) for session ${sessionKey}`,
+      );
+
+      let totalExtracted = 0;
+      let totalStored = 0;
+      let lastSceneName: string | undefined;
+
+      for (const group of groups) {
+        logger.debug?.(
+          `${TAG} [l1] Group sessionId=${group.sessionId || "(empty)"}: ${group.messages.length} messages`,
+        );
+
+        const l1Result = await extractL1Memories({
+          messages: group.messages,
+          sessionKey,
+          sessionId: group.sessionId,
+          baseDir: pluginDataDir,
+          config,
+          options: {
+            enableDedup: cfg.extraction.enableDedup,
+            maxMemoriesPerSession: cfg.extraction.maxMemoriesPerSession,
+            model: cfg.extraction.model,
+            previousSceneName: lastSceneName ?? (runnerState.last_scene_name || undefined),
+            vectorStore,
+            embeddingService,
+            conflictRecallTopK: cfg.embedding.conflictRecallTopK,
+          },
+          logger,
+          instanceId: getInstanceId?.(),
+        });
+
+        totalExtracted += l1Result.extractedCount;
+        totalStored += l1Result.storedCount;
+        if (l1Result.lastSceneName) {
+          lastSceneName = l1Result.lastSceneName;
+        }
+      }
+
+      // Use maxRecordedAtMs (write time) as cursor — always positive, TCVDB-safe
+      await checkpoint.markL1ExtractionComplete(sessionKey, totalStored, maxRecordedAtMs || undefined, lastSceneName);
+      logger.info(
+        `${TAG} [l1] L1 complete: extracted=${totalExtracted}, stored=${totalStored} (${groups.length} group(s))`,
+      );
+
+      return { processedCount: totalMessages };
+    } catch (err) {
+      logger.error(`${TAG} [l1] L1 failed: ${err instanceof Error ? err.stack ?? err.message : String(err)}`);
+      throw err;
+    }
+  };
+}
+
+// ============================
+// Persister factory
+// ============================
+
+/**
+ * Create the standard pipeline state persister.
+ * Saves pipeline session states to the checkpoint file.
+ */
+export function createPersister(
+  pluginDataDir: string,
+  logger: PipelineLogger,
+): (states: Record<string, PipelineSessionState>) => Promise<void> {
+  return async (states) => {
+    const checkpoint = new CheckpointManager(pluginDataDir, logger);
+    await checkpoint.mergePipelineStates(states);
+  };
+}
+
+// ============================
+// L2 Runner factory
+// ============================
+
+/**
+ * Create the standard L2 runner function (scene extraction).
+ *
+ * Reads L1 memory records (incremental via VectorStore or JSONL fallback),
+ * runs SceneExtractor, and returns the latest cursor for pipeline-manager
+ * to track incremental progress.
+ *
+ * Used by both `index.ts` (live runtime) and `seed-runtime.ts` (seed CLI).
+ */
+export function createL2Runner(opts: {
+  pluginDataDir: string;
+  cfg: MemoryTdaiConfig;
+  openclawConfig: unknown;
+  vectorStore: IMemoryStore | undefined;
+  logger: PipelineLogger;
+  instanceId?: string;
+}): L2Runner {
+  const { pluginDataDir, cfg, openclawConfig, vectorStore, logger, instanceId } = opts;
+  let profileBaseline = new Map<string, { version: number; contentMd5: string; createdAtMs: number }>();
+
+  return async (sessionKey: string, cursor?: string) => {
+    logger.debug?.(
+      `${TAG} [L2] session=${sessionKey}, updatedAfter=${cursor ?? "(full)"}`,
+    );
+
+    let records: Array<{ content: string; created_at: string; id: string; updatedAt: string }>;
+
+    if (vectorStore?.pullProfiles && !vectorStore.isDegraded()) {
+      profileBaseline = await pullProfilesToLocal(pluginDataDir, vectorStore, logger);
+    }
+
+    if (vectorStore && !vectorStore.isDegraded()) {
+      const { queryMemoryRecords } = await import("../record/l1-reader.js");
+      const memRecords = await queryMemoryRecords(vectorStore, {
+        sessionKey,
+        updatedAfter: cursor,
+      }, logger);
+
+      if (memRecords.length === 0) {
+        logger.debug?.(
+          `${TAG} [L2] No new L1 records since cursor (session=${sessionKey}, updatedAfter=${cursor ?? "(full)"}), skipping scene extraction`,
+        );
+        return;
+      }
+
+      logger.debug?.(
+        `${TAG} [L2] Incremental query returned ${memRecords.length} record(s) (session=${sessionKey})`,
+      );
+
+      records = memRecords.map((r) => ({
+        content: r.content,
+        created_at: r.createdAt,
+        id: r.id,
+        updatedAt: r.updatedAt,
+      }));
+    } else {
+      logger.debug?.(`${TAG} [L2] VectorStore unavailable, falling back to JSONL read (session=${sessionKey})`);
+      const { readMemoryRecords } = await import("../record/l1-reader.js");
+      let sessionRecords = await readMemoryRecords(sessionKey, pluginDataDir, logger);
+
+      if (cursor) {
+        const beforeCount = sessionRecords.length;
+        sessionRecords = sessionRecords.filter((r) => {
+          const t = r.updatedAt || r.createdAt || "";
+          return t > cursor;
+        });
+        logger.debug?.(
+          `${TAG} [L2] JSONL time filter: ${beforeCount} → ${sessionRecords.length} record(s) (updatedAfter=${cursor})`,
+        );
+      }
+
+      if (sessionRecords.length === 0) {
+        logger.debug?.(`${TAG} [L2] No new L1 records found (JSONL fallback, session=${sessionKey}), skipping scene extraction`);
+        return;
+      }
+
+      records = sessionRecords.map((r) => ({
+        content: r.content,
+        created_at: r.createdAt,
+        id: r.id,
+        updatedAt: r.updatedAt,
+      }));
+    }
+
+    const extractor = new SceneExtractor({
+      dataDir: pluginDataDir,
+      config: openclawConfig!,
+      model: cfg.persona.model,
+      maxScenes: cfg.persona.maxScenes,
+      sceneBackupCount: cfg.persona.sceneBackupCount,
+      logger,
+      instanceId,
+    });
+
+    const memories = records.map((r) => ({
+      content: r.content,
+      created_at: r.created_at,
+      id: r.id,
+    }));
+
+    const preCheckpoint = new CheckpointManager(pluginDataDir, logger);
+    const preState = await preCheckpoint.read();
+    const preScenesProcessed = preState.scenes_processed;
+    const preMemoriesSince = preState.memories_since_last_persona;
+    const preTotalProcessed = preState.total_processed;
+
+    const extractResult = await extractor.extract(memories);
+    if (extractResult.success && extractResult.memoriesProcessed > 0) {
+      const checkpoint = new CheckpointManager(pluginDataDir, logger);
+      const postState = await checkpoint.read();
+      if (
+        postState.scenes_processed < preScenesProcessed ||
+        postState.total_processed < preTotalProcessed
+      ) {
+        logger.warn(
+          `${TAG} [L2] ⚠️ Checkpoint corruption detected! ` +
+          `scenes_processed: ${preScenesProcessed} → ${postState.scenes_processed}, ` +
+          `total_processed: ${preTotalProcessed} → ${postState.total_processed}, ` +
+          `memories_since: ${preMemoriesSince} → ${postState.memories_since_last_persona}. ` +
+          `Repairing...`,
+        );
+        await checkpoint.write({
+          ...postState,
+          scenes_processed: Math.max(postState.scenes_processed, preScenesProcessed),
+          total_processed: Math.max(postState.total_processed, preTotalProcessed),
+          memories_since_last_persona: Math.max(postState.memories_since_last_persona, preMemoriesSince),
+        });
+        logger.info(`${TAG} [L2] Checkpoint repaired`);
+      }
+
+      if (vectorStore && supportsProfileSyncWrite(vectorStore)) {
+        await syncLocalProfilesToStore(pluginDataDir, vectorStore, profileBaseline, logger);
+      }
+      await checkpoint.incrementScenesProcessed();
+
+      const latestCursor = records.reduce((latest, r) => {
+        return r.updatedAt > latest ? r.updatedAt : latest;
+      }, "");
+
+      logger.debug?.(
+        `${TAG} [L2] Extraction complete: processed=${extractResult.memoriesProcessed}, latestCursor=${latestCursor}`,
+      );
+
+      return { latestCursor: latestCursor || undefined };
+    }
+  };
+}
+
+// ============================
+// L3 Runner factory
+// ============================
+
+/**
+ * Create the standard L3 runner function (persona generation).
+ *
+ * Uses PersonaTrigger to check if generation is needed, then runs
+ * PersonaGenerator. Used by both `index.ts` and `seed-runtime.ts`.
+ */
+export function createL3Runner(opts: {
+  pluginDataDir: string;
+  cfg: MemoryTdaiConfig;
+  openclawConfig: unknown;
+  vectorStore?: IMemoryStore;
+  logger: PipelineLogger;
+  instanceId?: string;
+}): L3Runner {
+  const { pluginDataDir, cfg, openclawConfig, vectorStore, logger, instanceId } = opts;
+
+  return async () => {
+    const trigger = new PersonaTrigger({
+      dataDir: pluginDataDir,
+      interval: cfg.persona.triggerEveryN,
+      logger,
+    });
+
+    const { should, reason } = await trigger.shouldGenerate();
+    if (!should) {
+      logger.debug?.(`${TAG} [L3] Persona generation not needed`);
+      return;
+    }
+
+    if (!openclawConfig) {
+      logger.warn(`${TAG} [L3] No OpenClaw config, skipping persona generation`);
+      return;
+    }
+
+    // Pull remote profiles to establish fresh baseline before generation.
+    // This ensures syncLocalProfilesToStore() has correct baselineVersion
+    // for the optimistic-lock check instead of defaulting to 0.
+    let profileBaseline = new Map<string, { version: number; contentMd5: string; createdAtMs: number }>();
+    if (vectorStore?.pullProfiles && !vectorStore.isDegraded()) {
+      profileBaseline = await pullProfilesToLocal(pluginDataDir, vectorStore, logger);
+    }
+
+    logger.info(`${TAG} [L3] Starting persona generation: ${reason}`);
+    const generator = new PersonaGenerator({
+      dataDir: pluginDataDir,
+      config: openclawConfig,
+      model: cfg.persona.model,
+      backupCount: cfg.persona.backupCount,
+      logger,
+      instanceId,
+    });
+    const genResult = await generator.generateLocalPersona(reason);
+    if (!genResult) {
+      logger.info(`${TAG} [L3] Persona generation skipped (no changes)`);
+      return;
+    }
+
+    if (vectorStore && supportsProfileSyncWrite(vectorStore)) {
+      await syncLocalProfilesToStore(pluginDataDir, vectorStore, profileBaseline, logger);
+    }
+
+    const checkpoint = new CheckpointManager(pluginDataDir, logger);
+    const cp = await checkpoint.read();
+    await checkpoint.markPersonaGenerated(cp.total_processed);
+    logger.info(`${TAG} [L3] Persona generation succeeded`);
+  };
+}
+
+// ============================
+// Pipeline Manager factory
+// ============================
+
+/**
+ * Create a MemoryPipelineManager with the standard config mapping.
+ */
+export function createPipelineManager(
+  cfg: MemoryTdaiConfig,
+  logger: PipelineLogger,
+  sessionFilter?: SessionFilter,
+): MemoryPipelineManager {
+  return new MemoryPipelineManager(
+    {
+      everyNConversations: cfg.pipeline.everyNConversations,
+      enableWarmup: cfg.pipeline.enableWarmup,
+      l1: { idleTimeoutSeconds: cfg.pipeline.l1IdleTimeoutSeconds },
+      l2: {
+        delayAfterL1Seconds: cfg.pipeline.l2DelayAfterL1Seconds,
+        minIntervalSeconds: cfg.pipeline.l2MinIntervalSeconds,
+        maxIntervalSeconds: cfg.pipeline.l2MaxIntervalSeconds,
+        sessionActiveWindowHours: cfg.pipeline.sessionActiveWindowHours,
+      },
+    },
+    logger,
+    sessionFilter ?? new SessionFilter([]),
+  );
+}
+
+// ============================
+// Full pipeline factory
+// ============================
+
+/**
+ * Create a fully wired pipeline instance: VectorStore + EmbeddingService +
+ * MemoryPipelineManager with L1 runner and persister attached.
+ *
+ * This is the high-level entry point used by both `index.ts` and `seed-runtime.ts`.
+ * Callers should attach L2/L3 runners after creation using `createL2Runner()`
+ * and `createL3Runner()` from this module.
+ */
+export async function createPipeline(opts: PipelineFactoryOptions): Promise<PipelineInstance> {
+  const { pluginDataDir, cfg, openclawConfig, logger, sessionFilter } = opts;
+
+  // Ensure data directories exist
+  initDataDirectories(pluginDataDir);
+
+  // Initialize stores (once-async: reuses cached result if already initialized)
+  const stores = await initStores(cfg, pluginDataDir, logger);
+  const { vectorStore, embeddingService } = stores;
+
+  // Create pipeline manager
+  const scheduler = createPipelineManager(cfg, logger, sessionFilter);
+
+  // Wire L1 runner
+  scheduler.setL1Runner(createL1Runner({
+    pluginDataDir,
+    cfg,
+    openclawConfig,
+    vectorStore,
+    embeddingService,
+    logger,
+  }));
+
+  // Wire persister
+  scheduler.setPersister(createPersister(pluginDataDir, logger));
+
+  // Destroy function
+  const destroy = async () => {
+    logger.info(`${TAG} Destroying pipeline...`);
+    await scheduler.destroy();
+    if (vectorStore) {
+      logger.info(`${TAG} Closing VectorStore`);
+      vectorStore.close();
+    }
+    if (embeddingService?.close) {
+      try {
+        logger.info(`${TAG} Closing EmbeddingService`);
+        await embeddingService.close();
+      } catch (err) {
+        logger.warn(`${TAG} Error closing EmbeddingService: ${err instanceof Error ? err.message : String(err)}`);
+      }
+    }
+    logger.info(`${TAG} Pipeline destroyed`);
+  };
+
+  return { scheduler, vectorStore, embeddingService, destroy };
+}