mirror of
https://github.com/TencentCloud/TencentDB-Agent-Memory
synced 2026-07-10 12:34:27 +00:00
fix(recall): truncate recalled context by code point, not code unit (#109)
truncateRecallLine capped lines with line.length and line.slice, which count and index by UTF-16 code unit. When recall.maxCharsPerMemory or recall.maxTotalRecallChars is set and the cut lands between the halves of a surrogate pair, the line keeps a lone surrogate that becomes U+FFFD once UTF-8 encoded for the request, corrupting any non-BMP character (emoji, CJK Ext-B) in the injected context. Slice on Array.from(line) code points instead. Both recall call sites route through this function. Same class as the sanitizeText fix in #31, reintroduced by the #71 budget path. Signed-off-by: Akhilesh Arora <akhildawra@gmail.com>
This commit is contained in:
@@ -776,11 +776,15 @@ function normalizeBudgetLimit(value: number | undefined): number | undefined {
|
||||
}
|
||||
|
||||
function truncateRecallLine(line: string, maxChars: number): string {
|
||||
if (line.length <= maxChars) return line;
|
||||
// Count and slice by code point, not UTF-16 code unit, so a cut never lands
|
||||
// between the halves of a surrogate pair (which would corrupt a non-BMP
|
||||
// character to U+FFFD when the line is UTF-8 encoded for the request).
|
||||
const cps = Array.from(line);
|
||||
if (cps.length <= maxChars) return line;
|
||||
if (maxChars <= RECALL_TRUNCATION_SUFFIX.length) {
|
||||
return line.slice(0, maxChars);
|
||||
return cps.slice(0, maxChars).join("");
|
||||
}
|
||||
return `${line.slice(0, maxChars - RECALL_TRUNCATION_SUFFIX.length).trimEnd()}${RECALL_TRUNCATION_SUFFIX}`;
|
||||
return `${cps.slice(0, maxChars - RECALL_TRUNCATION_SUFFIX.length).join("").trimEnd()}${RECALL_TRUNCATION_SUFFIX}`;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
Reference in New Issue
Block a user