fix(offload): preserve non-BMP characters in sanitizeText

UNSAFE_CHAR_RE included the surrogate range [\uD800-\uDFFF] without the `u`
flag, so JS treated strings as UTF-16 code units and stripped each half of
every well-formed non-BMP code point. sanitizeText and sanitizeJsonLine
therefore destroyed emoji, CJK Extension B, math bold, etc. in tool params,
tool results, and ref-md archives.

Adding the `u` flag makes paired surrogates combine into a single code point
before matching, so the [\uD800-\uDFFF] entry now matches only lone (malformed)
surrogates, which is the original intent. All other entries (replacement char,
C0/C1 controls, zero-width chars, line separators, BOM) keep their behavior.

Added a vitest suite covering the preserved and stripped cases.

Closes #30
This commit is contained in:
Akhilesh Arora
2026-05-16 22:23:45 +02:00
parent 5736acc871
commit bf58853343
2 changed files with 44 additions and 1 deletions
+43
View File
@@ -0,0 +1,43 @@
import { describe, expect, it } from "vitest";
import { sanitizeText } from "./storage.js";
describe("sanitizeText", () => {
it("preserves plain ASCII", () => {
expect(sanitizeText("hello world")).toBe("hello world");
});
it("preserves emoji and other non-BMP code points", () => {
// 🎉 = U+1F389, 𠮷 = U+20BB7 (CJK Extension B), 𝐀 = U+1D400 (math bold A).
// Each is a surrogate pair in UTF-16. Without the `u` flag, the
// [\uD800-\uDFFF] range in UNSAFE_CHAR_RE would strip each half
// independently and silently destroy these characters.
expect(sanitizeText("emoji \u{1F389} here")).toBe("emoji \u{1F389} here");
expect(sanitizeText("CJK ext-B \u{20BB7} here")).toBe(
"CJK ext-B \u{20BB7} here",
);
expect(sanitizeText("math bold \u{1D400} here")).toBe(
"math bold \u{1D400} here",
);
});
it("strips lone (malformed) surrogates", () => {
expect(sanitizeText("lone \uD800 surrogate")).toBe("lone surrogate");
expect(sanitizeText("lone \uDC00 surrogate")).toBe("lone surrogate");
});
it("strips C0 and C1 control characters", () => {
expect(sanitizeText("ctrlhere")).toBe("ctrlhere");
expect(sanitizeText("c1…here")).toBe("c1here");
});
it("strips zero-width characters and BOM", () => {
expect(sanitizeText("ab")).toBe("ab");
expect(sanitizeText("ab")).toBe("ab");
});
it("returns non-string input unchanged", () => {
// Matches the existing typeof guard in sanitizeText.
expect(sanitizeText(42 as unknown as string)).toBe(42);
});
});