From b6f33a283725645270bebd762d06437bfe855705 Mon Sep 17 00:00:00 2001 From: Mradul Singh Date: Sat, 21 Nov 2026 15:30:27 +0530 Subject: [PATCH] Decode invalid UTF-8 in short strings like TextDecoder utf8DecodeJs read continuation bytes without checking them or the string's end, so a truncated sequence took bytes from the next value and overlong sequences decoded to real characters (C0 80 -> NUL). Follow the WHATWG decoder: one U+FFFD per maximal subpart, and never read past byteLength. Fixes #305 --- src/utils/utf8.ts | 74 +++++++++++++++++++++-------- test/decode-invalid-utf8.test.ts | 81 ++++++++++++++++++++++++++++++++ 2 files changed, 135 insertions(+), 20 deletions(-) create mode 100644 test/decode-invalid-utf8.test.ts diff --git a/src/utils/utf8.ts b/src/utils/utf8.ts index 1494f70..8e11737 100644 --- a/src/utils/utf8.ts +++ b/src/utils/utf8.ts @@ -108,6 +108,9 @@ export function utf8Encode(str: string, output: Uint8Array, outputOffset: number const CHUNK_SIZE = 0x1_000; +// Decodes UTF-8 like TextDecoder (WHATWG Encoding Standard): an ill-formed +// sequence becomes one U+FFFD per maximal subpart, and a sequence is never +// read past `inputOffset + byteLength`. export function utf8DecodeJs(bytes: Uint8Array, inputOffset: number, byteLength: number): string { let offset = inputOffset; const end = offset + byteLength; @@ -119,29 +122,60 @@ export function utf8DecodeJs(bytes: Uint8Array, inputOffset: number, byteLength: if ((byte1 & 0x80) === 0) { // 1 byte units.push(byte1); - } else if ((byte1 & 0xe0) === 0xc0) { + } else if (byte1 >= 0xc2 && byte1 <= 0xdf) { // 2 bytes - const byte2 = bytes[offset++]! & 0x3f; - units.push(((byte1 & 0x1f) << 6) | byte2); - } else if ((byte1 & 0xf0) === 0xe0) { - // 3 bytes - const byte2 = bytes[offset++]! & 0x3f; - const byte3 = bytes[offset++]! & 0x3f; - units.push(((byte1 & 0x1f) << 12) | (byte2 << 6) | byte3); - } else if ((byte1 & 0xf8) === 0xf0) { - // 4 bytes - const byte2 = bytes[offset++]! & 0x3f; - const byte3 = bytes[offset++]! & 0x3f; - const byte4 = bytes[offset++]! & 0x3f; - let unit = ((byte1 & 0x07) << 0x12) | (byte2 << 0x0c) | (byte3 << 0x06) | byte4; - if (unit > 0xffff) { - unit -= 0x10000; - units.push(((unit >>> 10) & 0x3ff) | 0xd800); - unit = 0xdc00 | (unit & 0x3ff); + const byte2 = offset < end ? bytes[offset]! : 0; + if ((byte2 & 0xc0) === 0x80) { + offset++; + units.push(((byte1 & 0x1f) << 6) | (byte2 & 0x3f)); + } else { + units.push(0xfffd); + } + } else if (byte1 >= 0xe0 && byte1 <= 0xef) { + // 3 bytes; E0 and ED narrow the second byte to reject overlongs and surrogates + const byte2 = offset < end ? bytes[offset]! : 0; + const lower = byte1 === 0xe0 ? 0xa0 : 0x80; + const upper = byte1 === 0xed ? 0x9f : 0xbf; + if (byte2 >= lower && byte2 <= upper) { + offset++; + const byte3 = offset < end ? bytes[offset]! : 0; + if ((byte3 & 0xc0) === 0x80) { + offset++; + units.push(((byte1 & 0x0f) << 12) | ((byte2 & 0x3f) << 6) | (byte3 & 0x3f)); + } else { + units.push(0xfffd); + } + } else { + units.push(0xfffd); + } + } else if (byte1 >= 0xf0 && byte1 <= 0xf4) { + // 4 bytes; F0 and F4 narrow the second byte to reject overlongs and values above U+10FFFF + const byte2 = offset < end ? bytes[offset]! : 0; + const lower = byte1 === 0xf0 ? 0x90 : 0x80; + const upper = byte1 === 0xf4 ? 0x8f : 0xbf; + if (byte2 >= lower && byte2 <= upper) { + offset++; + const byte3 = offset < end ? bytes[offset]! : 0; + if ((byte3 & 0xc0) === 0x80) { + offset++; + const byte4 = offset < end ? bytes[offset]! : 0; + if ((byte4 & 0xc0) === 0x80) { + offset++; + const unit = + (((byte1 & 0x07) << 18) | ((byte2 & 0x3f) << 12) | ((byte3 & 0x3f) << 6) | (byte4 & 0x3f)) - 0x10000; + units.push(((unit >>> 10) & 0x3ff) | 0xd800, 0xdc00 | (unit & 0x3ff)); + } else { + units.push(0xfffd); + } + } else { + units.push(0xfffd); + } + } else { + units.push(0xfffd); } - units.push(unit); } else { - units.push(byte1); + // continuation byte without a lead byte, C0, C1 or F5..FF + units.push(0xfffd); } if (units.length >= CHUNK_SIZE) { diff --git a/test/decode-invalid-utf8.test.ts b/test/decode-invalid-utf8.test.ts new file mode 100644 index 0000000..085664f --- /dev/null +++ b/test/decode-invalid-utf8.test.ts @@ -0,0 +1,81 @@ +import assert from "assert"; +import { decode } from "../src/index.ts"; +import { utf8DecodeJs, utf8DecodeTD } from "../src/utils/utf8.ts"; + +function str8(bytes: Array): Array { + return [0xd9, bytes.length, ...bytes]; +} + +describe("decode invalid UTF-8 strings", () => { + it("does not read a truncated sequence past the end of the string", () => { + // ["\xe2\x82", "A"]: the first string must not take the header of the second one + const actual = decode(Uint8Array.from([0x92, ...str8([0xe2, 0x82]), ...str8([0x41])])); + assert.deepStrictEqual(actual, ["\ufffd", "A"]); + }); + + it("does not read a truncated sequence past the end of a map key", () => { + const actual = decode(Uint8Array.from([0x81, 0xa2, 0xe2, 0x82, 0xa1, 0x41])); + assert.deepStrictEqual(actual, Object.fromEntries([["\ufffd", "A"]])); + }); + + it("decodes overlong sequences to U+FFFD, not to NUL", () => { + assert.deepStrictEqual(decode(Uint8Array.from(str8([0xc0, 0x80]))), "\ufffd\ufffd"); + assert.deepStrictEqual(decode(Uint8Array.from(str8([0xe0, 0x80, 0x80]))), "\ufffd\ufffd\ufffd"); + assert.deepStrictEqual(decode(Uint8Array.from(str8([0xf0, 0x80, 0x80, 0x80]))), "\ufffd\ufffd\ufffd\ufffd"); + }); + + it("decodes short strings the same as long strings", () => { + const samples = [ + [0x80], + [0xff], + [0xc2], + [0xc1, 0xbf], + [0xe2, 0x82], + [0xe0, 0x9f, 0xbf], + [0xed, 0xa0, 0x80], + [0xf0, 0x9f, 0x98], + [0xf4, 0x90, 0x80, 0x80], + [0xe2, 0x82, 0xac, 0xf0, 0x9f, 0x98, 0x80], + ]; + for (const sample of samples) { + const bytes = Uint8Array.from(sample); + assert.strictEqual(utf8DecodeJs(bytes, 0, bytes.length), utf8DecodeTD(bytes, 0, bytes.length), String(sample)); + } + }); + + it("matches TextDecoder on random byte sequences", () => { + const pieces = [ + [0x41], + [0x7f], + [0xc2, 0xa9], + [0xe2, 0x82, 0xac], + [0xf0, 0x9f, 0x98, 0x80], + [0xf4, 0x8f, 0xbf, 0xbf], + [0xef, 0xbb, 0xbf], + [0xc0, 0x80], + [0xe0, 0x80, 0x80], + [0xed, 0xa0, 0x80], + [0xf4, 0x90, 0x80, 0x80], + [0xf5], + [0x80], + [0xc2], + [0xe2, 0x82], + [0xf0, 0x9f, 0x98], + ]; + const decoder = new TextDecoder("utf-8", { ignoreBOM: true }); + let seed = 1; + const random = (n: number) => { + seed = (seed * 1103515245 + 12345) & 0x7fffffff; + return seed % n; + }; + for (let i = 0; i < 5000; i++) { + const sample: Array = []; + const count = 1 + random(6); + for (let j = 0; j < count; j++) { + sample.push(...pieces[random(pieces.length)]!); + } + const bytes = Uint8Array.from([0x20, ...sample, 0x20]); + assert.strictEqual(utf8DecodeJs(bytes, 1, sample.length), decoder.decode(bytes.subarray(1, 1 + sample.length))); + } + }); +});