mirror of
https://github.com/TencentCloud/TencentDB-Agent-Memory
synced 2026-07-10 12:34:27 +00:00
fix(recall): truncate recalled context by code point, not code unit (#109)
truncateRecallLine capped lines with line.length and line.slice, which count and index by UTF-16 code unit. When recall.maxCharsPerMemory or recall.maxTotalRecallChars is set and the cut lands between the halves of a surrogate pair, the line keeps a lone surrogate that becomes U+FFFD once UTF-8 encoded for the request, corrupting any non-BMP character (emoji, CJK Ext-B) in the injected context. Slice on Array.from(line) code points instead. Both recall call sites route through this function. Same class as the sanitizeText fix in #31, reintroduced by the #71 budget path. Signed-off-by: Akhilesh Arora <akhildawra@gmail.com>
This commit is contained in:
@@ -776,11 +776,15 @@ function normalizeBudgetLimit(value: number | undefined): number | undefined {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function truncateRecallLine(line: string, maxChars: number): string {
|
function truncateRecallLine(line: string, maxChars: number): string {
|
||||||
if (line.length <= maxChars) return line;
|
// Count and slice by code point, not UTF-16 code unit, so a cut never lands
|
||||||
|
// between the halves of a surrogate pair (which would corrupt a non-BMP
|
||||||
|
// character to U+FFFD when the line is UTF-8 encoded for the request).
|
||||||
|
const cps = Array.from(line);
|
||||||
|
if (cps.length <= maxChars) return line;
|
||||||
if (maxChars <= RECALL_TRUNCATION_SUFFIX.length) {
|
if (maxChars <= RECALL_TRUNCATION_SUFFIX.length) {
|
||||||
return line.slice(0, maxChars);
|
return cps.slice(0, maxChars).join("");
|
||||||
}
|
}
|
||||||
return `${line.slice(0, maxChars - RECALL_TRUNCATION_SUFFIX.length).trimEnd()}${RECALL_TRUNCATION_SUFFIX}`;
|
return `${cps.slice(0, maxChars - RECALL_TRUNCATION_SUFFIX.length).join("").trimEnd()}${RECALL_TRUNCATION_SUFFIX}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
Reference in New Issue
Block a user