fix(pipeline): return cursor on zero-record L2 extraction for incremental processing (#130)

Fixes #98

Return { latestCursor: cursor } instead of void when VectorStore or
JSONL pipeline returns zero records. Add cold-start guard in pipeline
manager to initialize last_extraction_updated_time on first run.

Previously the L2 runner performed a full table scan on every invocation
because the cursor never advanced past the initial state.

Signed-off-by: Oxygen56 <1391083091@qq.com>
This commit is contained in:
Willow Lopez
2026-06-15 01:04:52 +08:00
committed by GitHub
parent a21ef3f66a
commit f61c5fd5fd
2 changed files with 7 additions and 2 deletions
+2 -2
View File
@@ -463,7 +463,7 @@ export function createL2Runner(opts: {
logger.debug?.( logger.debug?.(
`${TAG} [L2] No new L1 records since cursor (session=${sessionKey}, updatedAfter=${cursor ?? "(full)"}), skipping scene extraction`, `${TAG} [L2] No new L1 records since cursor (session=${sessionKey}, updatedAfter=${cursor ?? "(full)"}), skipping scene extraction`,
); );
return { skipped: true }; return { skipped: true, latestCursor: cursor || undefined };
} }
logger.debug?.( logger.debug?.(
@@ -494,7 +494,7 @@ export function createL2Runner(opts: {
if (sessionRecords.length === 0) { if (sessionRecords.length === 0) {
logger.debug?.(`${TAG} [L2] No new L1 records found (JSONL fallback, session=${sessionKey}), skipping scene extraction`); logger.debug?.(`${TAG} [L2] No new L1 records found (JSONL fallback, session=${sessionKey}), skipping scene extraction`);
return; return { latestCursor: cursor || undefined };
} }
records = sessionRecords.map((r) => ({ records = sessionRecords.map((r) => ({
+5
View File
@@ -923,6 +923,11 @@ export class MemoryPipelineManager {
// Advance cursor using the record timestamp returned by the runner // Advance cursor using the record timestamp returned by the runner
if (result?.latestCursor) { if (result?.latestCursor) {
state.last_extraction_updated_time = result.latestCursor; state.last_extraction_updated_time = result.latestCursor;
} else if (!state.last_extraction_updated_time) {
// Cold-start guard: if runner returned void (e.g. extraction failure) and
// last_extraction_updated_time is still empty, initialize it to now so
// the next L2 run doesn't do a full table scan.
state.last_extraction_updated_time = new Date().toISOString();
} }
await this.persistStates(); await this.persistStates();