mirror of
https://github.com/TencentCloud/TencentDB-Agent-Memory
synced 2026-07-10 12:34:27 +00:00
Akhilesh Arora
bf58853343
UNSAFE_CHAR_RE included the surrogate range [\uD800-\uDFFF] without the `u` flag, so JS treated strings as UTF-16 code units and stripped each half of every well-formed non-BMP code point. sanitizeText and sanitizeJsonLine therefore destroyed emoji, CJK Extension B, math bold, etc. in tool params, tool results, and ref-md archives. Adding the `u` flag makes paired surrogates combine into a single code point before matching, so the [\uD800-\uDFFF] entry now matches only lone (malformed) surrogates, which is the original intent. All other entries (replacement char, C0/C1 controls, zero-width chars, line separators, BOM) keep their behavior. Added a vitest suite covering the preserved and stripped cases. Closes #30
44 lines
1.5 KiB
TypeScript
44 lines
1.5 KiB
TypeScript
import { describe, expect, it } from "vitest";
|
||
|
||
import { sanitizeText } from "./storage.js";
|
||
|
||
describe("sanitizeText", () => {
|
||
it("preserves plain ASCII", () => {
|
||
expect(sanitizeText("hello world")).toBe("hello world");
|
||
});
|
||
|
||
it("preserves emoji and other non-BMP code points", () => {
|
||
// 🎉 = U+1F389, 𠮷 = U+20BB7 (CJK Extension B), 𝐀 = U+1D400 (math bold A).
|
||
// Each is a surrogate pair in UTF-16. Without the `u` flag, the
|
||
// [\uD800-\uDFFF] range in UNSAFE_CHAR_RE would strip each half
|
||
// independently and silently destroy these characters.
|
||
expect(sanitizeText("emoji \u{1F389} here")).toBe("emoji \u{1F389} here");
|
||
expect(sanitizeText("CJK ext-B \u{20BB7} here")).toBe(
|
||
"CJK ext-B \u{20BB7} here",
|
||
);
|
||
expect(sanitizeText("math bold \u{1D400} here")).toBe(
|
||
"math bold \u{1D400} here",
|
||
);
|
||
});
|
||
|
||
it("strips lone (malformed) surrogates", () => {
|
||
expect(sanitizeText("lone \uD800 surrogate")).toBe("lone surrogate");
|
||
expect(sanitizeText("lone \uDC00 surrogate")).toBe("lone surrogate");
|
||
});
|
||
|
||
it("strips C0 and C1 control characters", () => {
|
||
expect(sanitizeText("ctrlhere")).toBe("ctrlhere");
|
||
expect(sanitizeText("c1
here")).toBe("c1here");
|
||
});
|
||
|
||
it("strips zero-width characters and BOM", () => {
|
||
expect(sanitizeText("ab")).toBe("ab");
|
||
expect(sanitizeText("ab")).toBe("ab");
|
||
});
|
||
|
||
it("returns non-string input unchanged", () => {
|
||
// Matches the existing typeof guard in sanitizeText.
|
||
expect(sanitizeText(42 as unknown as string)).toBe(42);
|
||
});
|
||
});
|