Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
74 changes: 54 additions & 20 deletions src/utils/utf8.ts
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,9 @@ export function utf8Encode(str: string, output: Uint8Array, outputOffset: number

const CHUNK_SIZE = 0x1_000;

// Decodes UTF-8 like TextDecoder (WHATWG Encoding Standard): an ill-formed
// sequence becomes one U+FFFD per maximal subpart, and a sequence is never
// read past `inputOffset + byteLength`.
export function utf8DecodeJs(bytes: Uint8Array, inputOffset: number, byteLength: number): string {
let offset = inputOffset;
const end = offset + byteLength;
Expand All @@ -119,29 +122,60 @@ export function utf8DecodeJs(bytes: Uint8Array, inputOffset: number, byteLength:
if ((byte1 & 0x80) === 0) {
// 1 byte
units.push(byte1);
} else if ((byte1 & 0xe0) === 0xc0) {
} else if (byte1 >= 0xc2 && byte1 <= 0xdf) {
// 2 bytes
const byte2 = bytes[offset++]! & 0x3f;
units.push(((byte1 & 0x1f) << 6) | byte2);
} else if ((byte1 & 0xf0) === 0xe0) {
// 3 bytes
const byte2 = bytes[offset++]! & 0x3f;
const byte3 = bytes[offset++]! & 0x3f;
units.push(((byte1 & 0x1f) << 12) | (byte2 << 6) | byte3);
} else if ((byte1 & 0xf8) === 0xf0) {
// 4 bytes
const byte2 = bytes[offset++]! & 0x3f;
const byte3 = bytes[offset++]! & 0x3f;
const byte4 = bytes[offset++]! & 0x3f;
let unit = ((byte1 & 0x07) << 0x12) | (byte2 << 0x0c) | (byte3 << 0x06) | byte4;
if (unit > 0xffff) {
unit -= 0x10000;
units.push(((unit >>> 10) & 0x3ff) | 0xd800);
unit = 0xdc00 | (unit & 0x3ff);
const byte2 = offset < end ? bytes[offset]! : 0;
if ((byte2 & 0xc0) === 0x80) {
offset++;
units.push(((byte1 & 0x1f) << 6) | (byte2 & 0x3f));
} else {
units.push(0xfffd);
}
} else if (byte1 >= 0xe0 && byte1 <= 0xef) {
// 3 bytes; E0 and ED narrow the second byte to reject overlongs and surrogates
const byte2 = offset < end ? bytes[offset]! : 0;
const lower = byte1 === 0xe0 ? 0xa0 : 0x80;
const upper = byte1 === 0xed ? 0x9f : 0xbf;
if (byte2 >= lower && byte2 <= upper) {
offset++;
const byte3 = offset < end ? bytes[offset]! : 0;
if ((byte3 & 0xc0) === 0x80) {
offset++;
units.push(((byte1 & 0x0f) << 12) | ((byte2 & 0x3f) << 6) | (byte3 & 0x3f));
} else {
units.push(0xfffd);
}
} else {
units.push(0xfffd);
}
} else if (byte1 >= 0xf0 && byte1 <= 0xf4) {
// 4 bytes; F0 and F4 narrow the second byte to reject overlongs and values above U+10FFFF
const byte2 = offset < end ? bytes[offset]! : 0;
const lower = byte1 === 0xf0 ? 0x90 : 0x80;
const upper = byte1 === 0xf4 ? 0x8f : 0xbf;
if (byte2 >= lower && byte2 <= upper) {
offset++;
const byte3 = offset < end ? bytes[offset]! : 0;
if ((byte3 & 0xc0) === 0x80) {
offset++;
const byte4 = offset < end ? bytes[offset]! : 0;
if ((byte4 & 0xc0) === 0x80) {
offset++;
const unit =
(((byte1 & 0x07) << 18) | ((byte2 & 0x3f) << 12) | ((byte3 & 0x3f) << 6) | (byte4 & 0x3f)) - 0x10000;
units.push(((unit >>> 10) & 0x3ff) | 0xd800, 0xdc00 | (unit & 0x3ff));
} else {
units.push(0xfffd);
}
} else {
units.push(0xfffd);
}
} else {
units.push(0xfffd);
}
units.push(unit);
} else {
units.push(byte1);
// continuation byte without a lead byte, C0, C1 or F5..FF
units.push(0xfffd);
}

if (units.length >= CHUNK_SIZE) {
Expand Down
81 changes: 81 additions & 0 deletions test/decode-invalid-utf8.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
import assert from "assert";
import { decode } from "../src/index.ts";
import { utf8DecodeJs, utf8DecodeTD } from "../src/utils/utf8.ts";

function str8(bytes: Array<number>): Array<number> {
return [0xd9, bytes.length, ...bytes];
}

describe("decode invalid UTF-8 strings", () => {
it("does not read a truncated sequence past the end of the string", () => {
// ["\xe2\x82", "A"]: the first string must not take the header of the second one
const actual = decode(Uint8Array.from([0x92, ...str8([0xe2, 0x82]), ...str8([0x41])]));
assert.deepStrictEqual(actual, ["\ufffd", "A"]);
});

it("does not read a truncated sequence past the end of a map key", () => {
const actual = decode(Uint8Array.from([0x81, 0xa2, 0xe2, 0x82, 0xa1, 0x41]));
assert.deepStrictEqual(actual, Object.fromEntries([["\ufffd", "A"]]));
});

it("decodes overlong sequences to U+FFFD, not to NUL", () => {
assert.deepStrictEqual(decode(Uint8Array.from(str8([0xc0, 0x80]))), "\ufffd\ufffd");
assert.deepStrictEqual(decode(Uint8Array.from(str8([0xe0, 0x80, 0x80]))), "\ufffd\ufffd\ufffd");
assert.deepStrictEqual(decode(Uint8Array.from(str8([0xf0, 0x80, 0x80, 0x80]))), "\ufffd\ufffd\ufffd\ufffd");
});

it("decodes short strings the same as long strings", () => {
const samples = [
[0x80],
[0xff],
[0xc2],
[0xc1, 0xbf],
[0xe2, 0x82],
[0xe0, 0x9f, 0xbf],
[0xed, 0xa0, 0x80],
[0xf0, 0x9f, 0x98],
[0xf4, 0x90, 0x80, 0x80],
[0xe2, 0x82, 0xac, 0xf0, 0x9f, 0x98, 0x80],
];
for (const sample of samples) {
const bytes = Uint8Array.from(sample);
assert.strictEqual(utf8DecodeJs(bytes, 0, bytes.length), utf8DecodeTD(bytes, 0, bytes.length), String(sample));
}
});

it("matches TextDecoder on random byte sequences", () => {
const pieces = [
[0x41],
[0x7f],
[0xc2, 0xa9],
[0xe2, 0x82, 0xac],
[0xf0, 0x9f, 0x98, 0x80],
[0xf4, 0x8f, 0xbf, 0xbf],
[0xef, 0xbb, 0xbf],
[0xc0, 0x80],
[0xe0, 0x80, 0x80],
[0xed, 0xa0, 0x80],
[0xf4, 0x90, 0x80, 0x80],
[0xf5],
[0x80],
[0xc2],
[0xe2, 0x82],
[0xf0, 0x9f, 0x98],
];
const decoder = new TextDecoder("utf-8", { ignoreBOM: true });
let seed = 1;
const random = (n: number) => {
seed = (seed * 1103515245 + 12345) & 0x7fffffff;
return seed % n;
};
for (let i = 0; i < 5000; i++) {
const sample: Array<number> = [];
const count = 1 + random(6);
for (let j = 0; j < count; j++) {
sample.push(...pieces[random(pieces.length)]!);
}
const bytes = Uint8Array.from([0x20, ...sample, 0x20]);
assert.strictEqual(utf8DecodeJs(bytes, 1, sample.length), decoder.decode(bytes.subarray(1, 1 + sample.length)));
}
});
});